From 999a11a0dd0cc41f8f2314070c2126a83d725b71 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 6 Feb 2025 13:14:38 -0700 Subject: [PATCH 01/91] Merge adf_histogram.py from regrid_se_option branch --- scripts/plotting/adf_histogram.py | 290 ++++++++++++++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 scripts/plotting/adf_histogram.py diff --git a/scripts/plotting/adf_histogram.py b/scripts/plotting/adf_histogram.py new file mode 100644 index 000000000..2d8b58661 --- /dev/null +++ b/scripts/plotting/adf_histogram.py @@ -0,0 +1,290 @@ +""" +adf_histogram + +Create histogram(s) of all specified (2D) variables. + +- Constructs global histograms: total, ocean, land. + + for consistency with other ADF, will default to making annual and seasonal histograms. +- Saves the resulting histogram data as netCDF (into the plots directory by default) +- Plots the histograms. + +Options: + - specify whether to do spatial distribution from climo files + or combine temporal and spatial using time series files + +Possible future enhancements: + - specify a landmask file + - specific histogram options in the variable defaults file (looks for hist_bins) + - ability to specify additional regions (?) + - specify the output location for netCDF files + - improved detection of dimensions that (like cosp height) + - allow for rgridding; right now if LANDFRAC and data are not the same, just skips. + +""" + +from pathlib import Path +import numpy as np +import xarray as xr +import matplotlib.pyplot as plt +import plotting_functions as pf +import warnings # use to warn user about missing files. +def my_formatwarning(msg, *args, **kwargs): + # ignore everything except the message + return str(msg) + '\n' +warnings.formatwarning = my_formatwarning + +# +# USER-ADJUSTABLE PARAMETERS: +# + +# climo or time series: +use_time_series = True +# climo files are much smaller, so calculation will be faster, +# time series are usually more statistically robust. + +add_dimension_annotation = True # will print the dimensions of the input data on the plot + +redo_histogram_files = False + +if use_time_series: + plot_name_string = "GlobalHistogramTS" +else: + plot_name_string = "GlobalHistogramClimo" + +#Set seasonal ranges: +seasons = {"ANN": np.arange(1,13,1), + "DJF": [12, 1, 2], + "JJA": [6, 7, 8], + "MAM": [3, 4, 5], + "SON": [9, 10, 11]} + +def adf_histogram(adfobj): + var_list = adfobj.diag_var_list + + if use_time_series: + load_ref_func = adfobj.data.load_reference_timeseries_da # field [this is a mistake, the args should have been the same for all of these functions] + load_func = adfobj.data.load_timeseries_da # case, variablename + else: + load_ref_func = adfobj.data.load_reference_climo_da # case, variablename + load_func = adfobj.data.load_climo_da # case, variablename + + def get_load_args(adfobj, case, variablename): + if use_time_series and (case == adfobj.data.ref_case_label): + return (variablename, ) + else: + return (case, variablename) + + # Standard ADF stuff: + plot_locations = adfobj.plot_location + syear_cases = adfobj.climo_yrs["syears"] + eyear_cases = adfobj.climo_yrs["eyears"] + syear_baseline = adfobj.climo_yrs["syear_baseline"] + eyear_baseline = adfobj.climo_yrs["eyear_baseline"] + res = adfobj.variable_defaults + basic_info_dict = adfobj.read_config_var("diag_basic_info") + plot_type = basic_info_dict.get("plot_type", "png") + print(f"\t NOTE: Plot type is set to {plot_type}") + res = adfobj.variable_defaults + + # check if existing plots need to be redone + redo_plot = adfobj.get_basic_info("redo_plot") + print(f"\t NOTE: redo_plot is set to {redo_plot}") + # + # SECTION 1: determine which plots need to be made + # + for case_idx, case_name in enumerate(adfobj.data.case_names): + # Set output plot location: + plot_loc = Path(plot_locations[case_idx]) + print(f"PLOT LOCATION: {plot_loc}") + # Loop over the variables for each season + skip_make_plot = [] + for var in var_list: + for s in seasons: + plot_name = plot_loc / f"{var}_{s}_{plot_name_string}.{plot_type}" + plot_exists = plot_name.is_file() + print( + f"Projected file name: {plot_name.name}. Exists: {plot_exists}" + ) + if plot_exists and (not redo_plot): + skip_make_plot.append(plot_name) + # make histogram files for each variable + # output: variable(season, region, bin) + + # "reference case" first: + ref_land = load_ref_func(*get_load_args(adfobj, adfobj.data.ref_case_label, "LANDFRAC")) + for var in var_list: + + ref_hist_file = plot_loc / f"{adfobj.data.ref_case_label}_{var}_{plot_name_string}.nc" + ref_histogram_file_exists = ref_hist_file.is_file() + + if var in res: + vres = res[var] + adfobj.debug_log(f"adf_histogram: Found variable defaults for {var}") + else: + vres = {} + + # probably have to make sure no "lev" dim (but gets confused about other dimensions) + da = load_ref_func(*get_load_args(adfobj, adfobj.data.ref_case_label, var)) + if ("lev" in da.dims) or ("ilev" in da.dims): + print(f"{var}: Looks like lev/ilev present... skip") + continue + has_lat_lon = pf.lat_lon_validate_dims(da) + if not has_lat_lon: + print(f"INFO: {var} looks like it is on unstructured mesh. Has ncol: {'ncol' in da.dims}. Histogram does not need to regrid.") + + if (not ref_histogram_file_exists) or (redo_histogram_files): + ref_result = make_histograms(da, ref_land, vres) + print(f"Writing Reference Case Histogram: {ref_hist_file}") + ref_result.to_netcdf(ref_hist_file) + else: + print("reference histogram file exists, will use that one.") + + for case_idx, case_name in enumerate(adfobj.data.case_names): + case_land = load_func(*get_load_args(adfobj, case_name, "LANDFRAC")) + for var in var_list: + if var in res: + vres = res[var] + adfobj.debug_log(f"adf_histogram: Found variable defaults for {var}") + else: + vres = {} + hist_file = plot_loc / f"{case_name}_{var}_{plot_name_string}.nc" + histogram_file_exists = hist_file.is_file() + if (not histogram_file_exists) or (redo_histogram_files): + da = load_func(*get_load_args(adfobj, case_name, var)) + if ("lev" in da.dims) or ("ilev" in da.dims): + print(f"{var}: Looks like lev/ilev present... skip") + continue + has_lat_lon = pf.lat_lon_validate_dims(da) + if not has_lat_lon: + print(f"INFO: {var} looks like it is on unstructured mesh. Has ncol: {'ncol' in da.dims}. Histogram does not need to regrid.") + + result = make_histograms(da, case_land, vres) + print(f"Writing Case Histogram: {hist_file}") + result.to_netcdf(hist_file) + else: + print(f"{case_name} histogram file exists, will use that one.") + + print(f"HISTOGRAM PLOT LOCATION:\n{plot_loc}") + for var in var_list: + ref_hist_file = plot_loc / f"{adfobj.data.ref_case_label}_{var}_{plot_name_string}.nc" + ref_h_ds = xr.open_dataset(ref_hist_file)['histogram'] + add_annot = False + if "input_dims" in ref_h_ds.attrs: + add_annot = True + annot = f"input dimensions: {ref_h_ds.attrs['input_dims']}" + case_hist_files = [plot_loc / f"{case_name}_{var}_{plot_name_string}.nc" for case_name in adfobj.data.case_names] + case_h_ds = {c: xr.open_dataset(case_hist_files[i])['histogram'] for i, c in enumerate(adfobj.data.case_names)} + + for season in seasons: + fig, axes = plt.subplots(1, 3, figsize=(18, 6), sharey=True) + regions = ['ALL', 'OCEAN', 'LAND'] + for i, region in enumerate(regions): + ax = axes[i] + ref_hist = ref_h_ds.sel(season=season, region=region) + ax.step(ref_hist['bin'], ref_hist, where='mid', label=adfobj.data.ref_case_label, linestyle='--', color='gray') + for case_name, case_ds in case_h_ds.items(): + case_hist = case_ds.sel(season=season, region=region) + ax.step(case_hist['bin'], case_hist, where='mid', label=case_name) + ax.set_title(f"{region} - {season}") + ax.set_xlabel(var) + if i == 0: + ax.set_ylabel('Frequency') + ax.set_ylim([0,1]) + ax.legend() + fig.suptitle(f"{var} Histogram - {season}") + if add_annot: + # Add the annotation in the lower right corner + axes[-1].annotate(annot, + xy=(0.8, -0.02), # Coordinates of the point to annotate (lower right corner of the figure) + xycoords='figure fraction', # Coordinate system is the figure fraction + xytext=(-10, 10), # Offset the text from the point, in points + textcoords='offset points', + ha='right', # Horizontal alignment: right + va='bottom') # Vertical alignment: bottom + + plot_name = plot_loc / f"{var}_{season}_{plot_name_string}.{plot_type}" + fig.savefig(plot_name, bbox_inches='tight', dpi=72) + plt.close(fig) + print(f"\t Saved {var} Histogram for {season}: {plot_name.name}") + +def make_histograms(data, land, vres): + + do_region_masks = True + if land.shape != data.shape: + print("LAND and DATA are different shapes... will not do region masking.") + do_region_masks = False + + # determine the bins + if 'hist_bins' in vres: + bins = vres['hist_bins'] + elif 'contour_levels_range' in vres: + bins = np.arange(*vres['contour_levels_range']) + else: + print("WARNING: no sensible defaults found -- histogram will use 25 bins (bins may differ across cases)") + bins = np.linspace(data.min(), data.max(), 26) + + # extend bins to catch all values + hbins = np.insert(bins, 0, np.finfo(float).min) + hbins = np.append(hbins, np.finfo(float).max) + # Usually you would want bin center values like so: + # bin_centers = (hbins[:-1] + hbins[1:]) / 2 + # but if we use massive numbers for the endpoints, then make more sensible "centers": + bin_centers = bins[0:-1] + 0.5 * np.diff(bins) + bin_centers = np.insert(bin_centers, 0, bins[0] - 0.5 * (bins[1] - bins[0])) + bin_centers = np.append(bin_centers, bins[-1] + 0.5 * (bins[-1] - bins[-2])) + + # create masks + # seasons = ['DJF', 'MAM', 'JJA', 'SON', 'ANN'] + if use_time_series: + season_masks = { + 'DJF': (data.time.dt.month.isin([12, 1, 2])), + 'MAM': (data.time.dt.month.isin([3, 4, 5])), + 'JJA': (data.time.dt.month.isin([6, 7, 8])), + 'SON': (data.time.dt.month.isin([9, 10, 11])), + 'ANN': (np.ones_like(data.time) == 1) + } + else: + season_masks = { + 'DJF': (data.time.isin([12, 1, 2])), + 'MAM': (data.time.isin([3, 4, 5])), + 'JJA': (data.time.isin([6, 7, 8])), + 'SON': (data.time.isin([9, 10, 11])), + 'ANN': (np.ones_like(data.time) == 1) + } + + region_masks = { + 'ALL': (np.ones_like(land) == 1), + 'OCEAN': (land <= 0), + 'LAND': (land > 0) + } + + # calculate histograms + hist_data = np.zeros((len(seasons), len(region_masks), len(bin_centers))) + for i, season in enumerate(seasons): + for j, (region, rmask) in enumerate(region_masks.items()): + if do_region_masks: + masked_data = xr.where(rmask, data, np.nan) + else: + if region == 'ALL': + masked_data = data + else: + hist_data[i, j, :] = np.nan + continue + masked_data = masked_data.isel(time=season_masks[season]) # data.where(season_masks[season]) + hist, _ = np.histogram(masked_data, bins=hbins, density=True) + hist = hist * np.diff(hbins) # convert to probability mass function + hist_data[i, j, :] = hist + # create DataArray + histogram_da = xr.DataArray( + hist_data, + dims=['season', 'region', 'bin'], + coords={ + 'season': list(seasons.keys()), + 'region': list(region_masks.keys()), + 'bin': bin_centers, + }, + attrs={'input_dims':f"({','.join(list(data.dims))})"}, + name='histogram' + ) + + return histogram_da \ No newline at end of file From 7c0539f047fa8a0d618b4297391d3d3a7b6e76dd Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 6 Feb 2025 14:58:20 -0700 Subject: [PATCH 02/91] first attempt to get histograms on web output --- lib/adf_variable_defaults.yaml | 2 +- scripts/plotting/adf_histogram.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index 60743359c..c94265469 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -69,7 +69,7 @@ # Available ADF Default Plot Types #+++++++++++++ default_ptypes: ["Tables","LatLon","LatLon_Vector","Zonal","Meridional", - "NHPolar","SHPolar","TimeSeries","Special"] + "NHPolar","SHPolar","TimeSeries","Histogram","Special"] #+++++++++++++ # Constants diff --git a/scripts/plotting/adf_histogram.py b/scripts/plotting/adf_histogram.py index 2d8b58661..31475e01b 100644 --- a/scripts/plotting/adf_histogram.py +++ b/scripts/plotting/adf_histogram.py @@ -205,6 +205,14 @@ def get_load_args(adfobj, case, variablename): plot_name = plot_loc / f"{var}_{season}_{plot_name_string}.{plot_type}" fig.savefig(plot_name, bbox_inches='tight', dpi=72) plt.close(fig) + adfobj.add_website_data( + plot_name, + var, + None, + season=season, + multi_case=True, + plot_type="Histogram", + ) print(f"\t Saved {var} Histogram for {season}: {plot_name.name}") def make_histograms(data, land, vres): From 4c60852b5e27b979df855c12961a22af1735ddfb Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 6 Feb 2025 15:10:01 -0700 Subject: [PATCH 03/91] fixing web page generation --- lib/adf_variable_defaults.yaml | 2 +- scripts/plotting/adf_histogram.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index c94265469..f1da0600e 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -69,7 +69,7 @@ # Available ADF Default Plot Types #+++++++++++++ default_ptypes: ["Tables","LatLon","LatLon_Vector","Zonal","Meridional", - "NHPolar","SHPolar","TimeSeries","Histogram","Special"] + "NHPolar","SHPolar","TimeSeries","GlobalHistogramTS", "GlobalHistogramClimo","Special"] #+++++++++++++ # Constants diff --git a/scripts/plotting/adf_histogram.py b/scripts/plotting/adf_histogram.py index 31475e01b..8ebaab55f 100644 --- a/scripts/plotting/adf_histogram.py +++ b/scripts/plotting/adf_histogram.py @@ -100,7 +100,7 @@ def get_load_args(adfobj, case, variablename): skip_make_plot = [] for var in var_list: for s in seasons: - plot_name = plot_loc / f"{var}_{s}_{plot_name_string}.{plot_type}" + plot_name = plot_loc / f"{var}_{s}_{plot_name_string}_Mean.{plot_type}" plot_exists = plot_name.is_file() print( f"Projected file name: {plot_name.name}. Exists: {plot_exists}" @@ -202,7 +202,7 @@ def get_load_args(adfobj, case, variablename): ha='right', # Horizontal alignment: right va='bottom') # Vertical alignment: bottom - plot_name = plot_loc / f"{var}_{season}_{plot_name_string}.{plot_type}" + plot_name = plot_loc / f"{var}_{season}_{plot_name_string}_Mean.{plot_type}" fig.savefig(plot_name, bbox_inches='tight', dpi=72) plt.close(fig) adfobj.add_website_data( @@ -211,7 +211,7 @@ def get_load_args(adfobj, case, variablename): None, season=season, multi_case=True, - plot_type="Histogram", + plot_type=plot_name_string, ) print(f"\t Saved {var} Histogram for {season}: {plot_name.name}") From c411830374b52f881b90c01b8a0f91a5e66cd7e5 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 6 Feb 2025 15:35:35 -0700 Subject: [PATCH 04/91] provide some bins for OMEGA500 --- lib/adf_variable_defaults.yaml | 4 ++++ scripts/plotting/adf_histogram.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index f1da0600e..ba7a7152f 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -1877,6 +1877,10 @@ OMEGA500: category: "State" pct_diff_contour_levels: [-100,-75,-50,-40,-30,-20,-10,-8,-6,-4,-2,0,2,4,6,8,10,20,30,40,50,75,100] pct_diff_colormap: "PuOr_r" + scale_factor: 864 + add_offset: 0 + new_unit: "hPa d$^{-1}$" + hist_bins: [-105, -100, -95, -90, -85, -80, -75, -70, -65, -60, -55, -50, -45, -40, -35, -30, -25, -20, -15, -10, -5, 0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100] PINT: category: "State" diff --git a/scripts/plotting/adf_histogram.py b/scripts/plotting/adf_histogram.py index 8ebaab55f..7f5ff6cac 100644 --- a/scripts/plotting/adf_histogram.py +++ b/scripts/plotting/adf_histogram.py @@ -44,7 +44,7 @@ def my_formatwarning(msg, *args, **kwargs): add_dimension_annotation = True # will print the dimensions of the input data on the plot -redo_histogram_files = False +redo_histogram_files = True if use_time_series: plot_name_string = "GlobalHistogramTS" From e2822336739d52c7a7ecb8d9d2adf7591270c01a Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 6 Mar 2025 10:58:51 -0700 Subject: [PATCH 05/91] histogram debugging --- scripts/plotting/adf_histogram.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/plotting/adf_histogram.py b/scripts/plotting/adf_histogram.py index 7f5ff6cac..bad3dc5a5 100644 --- a/scripts/plotting/adf_histogram.py +++ b/scripts/plotting/adf_histogram.py @@ -125,9 +125,14 @@ def get_load_args(adfobj, case, variablename): # probably have to make sure no "lev" dim (but gets confused about other dimensions) da = load_ref_func(*get_load_args(adfobj, adfobj.data.ref_case_label, var)) + if da is None: + print(f"failed to load {var} for {adfobj.data.ref_case_label}... skip") + continue + if ("lev" in da.dims) or ("ilev" in da.dims): print(f"{var}: Looks like lev/ilev present... skip") continue + has_lat_lon = pf.lat_lon_validate_dims(da) if not has_lat_lon: print(f"INFO: {var} looks like it is on unstructured mesh. Has ncol: {'ncol' in da.dims}. Histogram does not need to regrid.") @@ -151,6 +156,9 @@ def get_load_args(adfobj, case, variablename): histogram_file_exists = hist_file.is_file() if (not histogram_file_exists) or (redo_histogram_files): da = load_func(*get_load_args(adfobj, case_name, var)) + if da is None: + print(f"Failed to load {var} for {case_name}... skip") + continue if ("lev" in da.dims) or ("ilev" in da.dims): print(f"{var}: Looks like lev/ilev present... skip") continue @@ -167,12 +175,18 @@ def get_load_args(adfobj, case, variablename): print(f"HISTOGRAM PLOT LOCATION:\n{plot_loc}") for var in var_list: ref_hist_file = plot_loc / f"{adfobj.data.ref_case_label}_{var}_{plot_name_string}.nc" + if not ref_hist_file.is_file(): + print(f"ERROR: histogram file not found for {var}, {adfobj.data.ref_case_label}... skip.") + continue ref_h_ds = xr.open_dataset(ref_hist_file)['histogram'] add_annot = False if "input_dims" in ref_h_ds.attrs: add_annot = True annot = f"input dimensions: {ref_h_ds.attrs['input_dims']}" case_hist_files = [plot_loc / f"{case_name}_{var}_{plot_name_string}.nc" for case_name in adfobj.data.case_names] + if not all([f.is_file() for f in case_hist_files]): + print(f"ERROR: histogram files not found for {var}, {adfobj.data.case_names}... skip") + continue case_h_ds = {c: xr.open_dataset(case_hist_files[i])['histogram'] for i, c in enumerate(adfobj.data.case_names)} for season in seasons: From a4d192b7c9a7dabcdae0567026be37d1a3a51556 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 6 Mar 2025 14:48:44 -0700 Subject: [PATCH 06/91] change for latlon maps, move to new branch --- scripts/plotting/global_latlon_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plotting/global_latlon_map.py b/scripts/plotting/global_latlon_map.py index ab37eb274..c44fb20f9 100644 --- a/scripts/plotting/global_latlon_map.py +++ b/scripts/plotting/global_latlon_map.py @@ -238,7 +238,7 @@ def global_latlon_map(adfobj): for s in seasons: plot_name = plot_loc / f"{var}_{pres}hpa_{s}_LatLon_Mean.{plot_type}" doplot[plot_name] = plot_file_op(adfobj, plot_name, f"{var}_{pres}hpa", case_name, s, web_category, redo_plot, "LatLon") - if all(value is None for value in doplot.values()): + if not any(value is None for value in doplot.values()): print(f"\t INFO: All plots exist for {var}. Redo is {redo_plot}. Existing plots added to website data. Continue.") continue From ff23690bc114f48d2c1e7b5a06a3485f1d99ab71 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 6 Mar 2025 14:55:26 -0700 Subject: [PATCH 07/91] fix logic for redo check --- scripts/plotting/global_latlon_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plotting/global_latlon_map.py b/scripts/plotting/global_latlon_map.py index ab37eb274..c44fb20f9 100644 --- a/scripts/plotting/global_latlon_map.py +++ b/scripts/plotting/global_latlon_map.py @@ -238,7 +238,7 @@ def global_latlon_map(adfobj): for s in seasons: plot_name = plot_loc / f"{var}_{pres}hpa_{s}_LatLon_Mean.{plot_type}" doplot[plot_name] = plot_file_op(adfobj, plot_name, f"{var}_{pres}hpa", case_name, s, web_category, redo_plot, "LatLon") - if all(value is None for value in doplot.values()): + if not any(value is None for value in doplot.values()): print(f"\t INFO: All plots exist for {var}. Redo is {redo_plot}. Existing plots added to website data. Continue.") continue From 4830a66c6351099554dc3b3e15d1574fec1bf47a Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 6 Mar 2025 17:06:48 -0700 Subject: [PATCH 08/91] initial fix and refactor fo polar_map.py --- scripts/plotting/polar_map.py | 442 +++++++++++----------------------- 1 file changed, 140 insertions(+), 302 deletions(-) diff --git a/scripts/plotting/polar_map.py b/scripts/plotting/polar_map.py index dbcfcff70..14fb9be1f 100644 --- a/scripts/plotting/polar_map.py +++ b/scripts/plotting/polar_map.py @@ -7,15 +7,29 @@ # ADF library import plotting_functions as pf +def get_hemisphere(hemi_type): + """Helper function to convert plot type to hemisphere code.""" + return "NH" if hemi_type == "NHPolar" else "SH" + +def process_seasonal_data(mdata, odata, season, vres): + """Helper function to calculate seasonal means and differences.""" + mseason = pf.seasonal_mean(mdata, season=season, is_climo=True) + oseason = pf.seasonal_mean(odata, season=season, is_climo=True) + + # Calculate differences + dseason = mseason - oseason + dseason.attrs['units'] = mseason.attrs['units'] + + # Calculate percent change + pseason = (mseason - oseason) / np.abs(oseason) * 100.0 + pseason.attrs['units'] = '%' + pseason = pseason.where(np.isfinite(pseason), np.nan) + pseason = pseason.fillna(0.0) + + return mseason, oseason, dseason, pseason + def polar_map(adfobj): - """ - This script/function generates polar maps of model fields with continental overlays. - Plots style follows old AMWG diagnostics: - - plots for ANN, DJF, MAM, JJA, SON - - separate files for each hemisphere, denoted `_nh` and `_sh` in file names. - - mean files shown on top row, difference on bottom row (centered) - [based on global_latlon_map.py] - """ + """Generate polar maps of model fields with continental overlays.""" #Notify user that script has started: msg = "\n Generating polar maps..." print(f"{msg}\n {'-' * (len(msg)-3)}") @@ -104,7 +118,6 @@ def polar_map(adfobj): # probably want to do this one variable at a time: for var in var_list: - #Notify user of variable being plotted: print(f"\t - polar maps for {var}") if var not in adfobj.data.ref_var_nam: @@ -113,310 +126,135 @@ def polar_map(adfobj): print(dmsg) continue - if adfobj.compare_obs: - #Check if obs exist for the variable: - if var in var_obs_dict: - #Note: In the future these may all be lists, but for - #now just convert the target_list. - #Extract target file: - dclimo_loc = var_obs_dict[var]["obs_file"] - #Extract target list (eventually will be a list, for now need to convert): - data_list = [var_obs_dict[var]["obs_name"]] - #Extract target variable name: - data_var = var_obs_dict[var]["obs_var"] - else: - dmsg = f"\t WARNING: No obs found for variable `{var}`, polar map skipped." - adfobj.debug_log(dmsg) - continue + if not adfobj.compare_obs: + base_name = adfobj.data.ref_labels[var] else: - #Set "data_var" for consistent use below: - data_var = var - #End if - - # Check res for any variable specific options that need to be used BEFORE going to the plot: - if var in res: - vres = res[var] - #If found then notify user, assuming debug log is enabled: - adfobj.debug_log(f"polar_map: Found variable defaults for {var}") + base_name = adfobj.data.ref_case_label + + + # Get variable-specific settings + vres = res.get(var, {}) + web_category = vres.get("category", None) + + # Get all plot info and check existence + plot_info = [] + all_plots_exist = True + + for case_idx, case_name in enumerate(case_names): + plot_loc = Path(plot_locations[case_idx]) + + for s in seasons: + for hemi_type in ["NHPolar", "SHPolar"]: + if pres_levs: + for pres in pres_levs: + plot_name = plot_loc / f"{var}_{pres}hpa_{s}_{hemi_type}_Mean.{plot_type}" + info = { + 'path': plot_name, + 'var': f"{var}_{pres}hpa", + 'case': case_name, + 'case_idx': case_idx, + 'season': s, + 'type': hemi_type, + 'pressure': pres, + 'exists': plot_name.is_file() + } + plot_info.append(info) + if not (redo_plot or not info['exists']): + adfobj.add_website_data(info['path'], info['var'], + info['case'], category=web_category, + season=s, plot_type=hemi_type) + else: + all_plots_exist = False + else: + plot_name = plot_loc / f"{var}_{s}_{hemi_type}_Mean.{plot_type}" + info = { + 'path': plot_name, + 'var': var, + 'case': case_name, + 'case_idx': case_idx, + 'season': s, + 'type': hemi_type, + 'exists': plot_name.is_file() + } + plot_info.append(info) + if not (redo_plot or not info['exists']): + adfobj.add_website_data(info['path'], info['var'], + info['case'], category=web_category, + season=s, plot_type=hemi_type) + else: + all_plots_exist = False + + if all_plots_exist: + print(f"\t Skipping {var} - all plots already exist") + continue - #Extract category (if available): - web_category = vres.get("category", None) + odata = adfobj.data.load_reference_regrid_da(base_name, var) + if odata is None: + print(f"\t WARNING: No reference data found for {var}") + continue - else: - vres = {} - web_category = None - #End if - - #loop over different data sets to plot model against: - for data_src in data_list: - - # load data (observational) commparison files (we should explore intake as an alternative to having this kind of repeated code): - if adfobj.compare_obs: - #For now, only grab one file (but convert to list for use below) - oclim_fils = [dclimo_loc] - #Set data name: - data_name = data_src - else: - oclim_fils = sorted(dclimo_loc.glob(f"{data_src}_{var}_baseline.nc")) - - oclim_ds = pf.load_dataset(oclim_fils) - if oclim_ds is None: - print("\t WARNING: Did not find any regridded reference climo files. Will try to skip.") - print(f"\t INFO: Data Location, dclimo_loc is {dclimo_loc}") - print(f"\t The glob is: {data_src}_{var}_*.nc") + # Process each case + for plot in plot_info: + if plot['exists'] and not redo_plot: continue + + case_name = plot['case'] + case_idx = plot['case_idx'] + plot_loc = Path(plot_locations[case_idx]) - #Loop over model cases: - for case_idx, case_name in enumerate(case_names): + # Ensure plot directory exists + plot_loc.mkdir(parents=True, exist_ok=True) - #Set case nickname: - case_nickname = test_nicknames[case_idx] - - #Set output plot location: - plot_loc = Path(plot_locations[case_idx]) - - #Check if plot output directory exists, and if not, then create it: - if not plot_loc.is_dir(): - print(f" {plot_loc} not found, making new directory") - plot_loc.mkdir(parents=True) + # Load and validate model data (units transformation included in load_regrid_da) + mdata = adfobj.data.load_regrid_da(case_name, var) + if mdata is None: + continue - # load re-gridded model files: - mclim_fils = sorted(mclimo_rg_loc.glob(f"{data_src}_{case_name}_{var}_*.nc")) + # Process data based on dimensionality + if "lev" in mdata.dims: + has_lev = True + else: + has_lev = False - mclim_ds = pf.load_dataset(mclim_fils) - if mclim_ds is None: - print("\t WARNING: Did not find any regridded test climo files. Will try to skip.") - print(f"\t INFO: Data Location, mclimo_rg_loc, is {mclimo_rg_loc}") - print(f"\t The glob is: {data_src}_{case_name}_{var}_*.nc") + if has_lev and pres_levs and plot.get('pressure'): + print("PRESSURE") + if not all(dim in mdata.dims for dim in ['lat', 'lev']): + continue + mdata = mdata.sel(lev=plot['pressure']) + odata_level = odata.sel(lev=plot['pressure']) + else: + print(f"EXPECT 2D VARIABLE: {odata.shape}") + if not pf.lat_lon_validate_dims(mdata): continue - #End if - - #Extract variable of interest - odata = oclim_ds[data_var].squeeze() # squeeze in case of degenerate dimensions - mdata = mclim_ds[var].squeeze() - - # APPLY UNITS TRANSFORMATION IF SPECIFIED: - # NOTE: looks like our climo files don't have all their metadata - mdata = mdata * vres.get("scale_factor",1) + vres.get("add_offset", 0) - # update units - mdata.attrs['units'] = vres.get("new_unit", mdata.attrs.get('units', 'none')) - - # Do the same for the baseline case if need be: - if not adfobj.compare_obs: - odata = odata * vres.get("scale_factor",1) + vres.get("add_offset", 0) - # update units - odata.attrs['units'] = vres.get("new_unit", odata.attrs.get('units', 'none')) - # or for observations. - else: - odata = odata * vres.get("obs_scale_factor",1) + vres.get("obs_add_offset", 0) - # Note: assume obs are set to have same untis as model. - - #Determine dimensions of variable: - has_dims = pf.lat_lon_validate_dims(odata) - if has_dims: - #If observations/baseline CAM have the correct - #dimensions, does the input CAM run have correct - #dimensions as well? - has_dims_cam = pf.lat_lon_validate_dims(mdata) - - #If both fields have the required dimensions, then - #proceed with plotting: - if has_dims_cam: - - # - # Seasonal Averages - # Note: xarray can do seasonal averaging, - # but depends on having time accessor, - # which these prototype climo files do not have. - # - - #Create new dictionaries: - mseasons = {} - oseasons = {} - dseasons = {} # hold the differences - pseasons = {} # hold percent change - - #Loop over season dictionary: - for s in seasons: - mseasons[s] = pf.seasonal_mean(mdata, season=s, is_climo=True) - oseasons[s] = pf.seasonal_mean(odata, season=s, is_climo=True) - # difference: each entry should be (lat, lon) - dseasons[s] = mseasons[s] - oseasons[s] - dseasons[s].attrs['units'] = mseasons[s].attrs['units'] - - # percent change - pseasons[s] = (mseasons[s] - oseasons[s]) / np.abs(oseasons[s]) * 100.0 # relative change - pseasons[s].attrs['units'] = '%' - #check if pct has NaN's or Inf values and if so set them to 0 to prevent plotting errors - pseasons[s] = pseasons[s].where(np.isfinite(pseasons[s]), np.nan) - pseasons[s] = pseasons[s].fillna(0.0) - - # make plots: northern and southern hemisphere separately: - for hemi_type in ["NHPolar", "SHPolar"]: - - #Create plot name and path: - plot_name = plot_loc / f"{var}_{s}_{hemi_type}_Mean.{plot_type}" - - # If redo_plot set to True: remove old plot, if it already exists: - if (not redo_plot) and plot_name.is_file(): - #Add already-existing plot to website (if enabled): - adfobj.debug_log(f"'{plot_name}' exists and clobber is false.") - adfobj.add_website_data(plot_name, var, case_name, category=web_category, - season=s, plot_type=hemi_type) - - #Continue to next iteration: - continue - else: - if plot_name.is_file(): - plot_name.unlink() - - #Create new plot: - # NOTE: send vres as kwarg dictionary. --> ONLY vres, not the full res - # This relies on `plot_map_and_save` knowing how to deal with the options - # currently knows how to handle: - # colormap, contour_levels, diff_colormap, diff_contour_levels, tiString, tiFontSize, mpl - # *Any other entries will be ignored. - # NOTE: If we were doing all the plotting here, we could use whatever we want from the provided YAML file. - - #Determine hemisphere to plot based on plot file name: - if hemi_type == "NHPolar": - hemi = "NH" - else: - hemi = "SH" - #End if - - pf.make_polar_plot(plot_name, case_nickname, base_nickname, - [syear_cases[case_idx],eyear_cases[case_idx]], - [syear_baseline,eyear_baseline], - mseasons[s], oseasons[s], dseasons[s], pseasons[s], hemisphere=hemi, obs=obs, **vres) - - #Add plot to website (if enabled): - adfobj.add_website_data(plot_name, var, case_name, category=web_category, - season=s, plot_type=hemi_type) - - else: #mdata dimensions check - print(f"\t WARNING: skipping polar map for {var} as it doesn't have only lat/lon dims.") - #End if (dimensions check) - - elif pres_levs: #Is the user wanting to interpolate to a specific pressure level? - - #Check that case inputs have the correct dimensions (including "lev"): - has_lat, has_lev = pf.zm_validate_dims(mdata) # assumes will work for both mdata & odata - - # check if there is a lat dimension: - if not has_lat: - print( - f"\t WARNING: Variable {var} is missing a lat dimension for '{case_name}', cannot continue to plot." - ) - continue - # End if - - #Check that case inputs have the correct dimensions (including "lev"): - has_lat_ref, has_lev_ref = pf.zm_validate_dims(odata) - - # check if there is a lat dimension: - if not has_lat_ref: - print( - f"\t WARNING: Variable {var} is missing a lat dimension for '{data_name}', cannot continue to plot." - ) - continue - - #Check if both cases have vertical levels to continue - if (has_lev) and (has_lev_ref): - - #Loop over pressure levels: - for pres in pres_levs: - - #Check that the user-requested pressure level - #exists in the model data, which should already - #have been interpolated to the standard reference - #pressure levels: - if not (pres in mclim_ds['lev']): - #Move on to the next pressure level: - print(f"\t WARNING: plot_press_levels value '{pres}' not a standard reference pressure, so skipping.") - continue - #End if - - #Create new dictionaries: - mseasons = {} - oseasons = {} - dseasons = {} # hold the differences - pseasons = {} # hold percent change - - #Loop over season dictionary: - for s in seasons: - mseasons[s] = (pf.seasonal_mean(mdata, season=s, is_climo=True)).sel(lev=pres) - oseasons[s] = (pf.seasonal_mean(odata, season=s, is_climo=True)).sel(lev=pres) - # difference: each entry should be (lat, lon) - dseasons[s] = mseasons[s] - oseasons[s] - dseasons[s].attrs['units'] = mseasons[s].attrs['units'] - - # percent change - pseasons[s] = (mseasons[s] - oseasons[s]) / abs(oseasons[s]) * 100.0 # relative change - pseasons[s].attrs['units'] = '%' - #check if pct has NaN's or Inf values and if so set them to 0 to prevent plotting errors - pseasons[s] = pseasons[s].where(np.isfinite(pseasons[s]), np.nan) - pseasons[s] = pseasons[s].fillna(0.0) - - # make plots: northern and southern hemisphere separately: - for hemi_type in ["NHPolar", "SHPolar"]: - - #Create plot name and path: - plot_name = plot_loc / f"{var}_{pres}hpa_{s}_{hemi_type}_Mean.{plot_type}" - - # If redo_plot set to True: remove old plot, if it already exists: - if (not redo_plot) and plot_name.is_file(): - #Add already-existing plot to website (if enabled): - adfobj.debug_log(f"'{plot_name}' exists and clobber is false.") - adfobj.add_website_data(plot_name, f"{var}_{pres}hpa", - case_name, category=web_category, - season=s, plot_type=hemi_type) - - #Continue to next iteration: - continue - else: - if plot_name.is_file(): - plot_name.unlink() - - #Create new plot: - # NOTE: send vres as kwarg dictionary. --> ONLY vres, not the full res - # This relies on `plot_map_and_save` knowing how to deal with the options - # currently knows how to handle: - # colormap, contour_levels, diff_colormap, diff_contour_levels, tiString, tiFontSize, mpl - # *Any other entries will be ignored. - # NOTE: If we were doing all the plotting here, we could use whatever we want from the provided YAML file. - - #Determine hemisphere to plot based on plot file name: - if hemi_type == "NHPolar": - hemi = "NH" - else: - hemi = "SH" - #End if - - pf.make_polar_plot(plot_name, case_nickname, base_nickname, - [syear_cases[case_idx],eyear_cases[case_idx]], - [syear_baseline,eyear_baseline], - mseasons[s], oseasons[s], dseasons[s], pseasons[s], hemisphere=hemi, obs=obs, **vres) - - #Add plot to website (if enabled): - adfobj.add_website_data(plot_name, f"{var}_{pres}hpa", - case_name, category=web_category, - season=s, plot_type=hemi_type) - - #End for (seasons) - #End for (pressure level) - else: - print(f"\t WARNING: variable '{var}' has no vertical dimension but is not just time/lat/lon, so skipping.") - #End if (has_lev) - else: #odata dimensions check - print(f"\t WARNING: skipping polar map for {var} as it has more than lat/lon dims, but no pressure levels were provided") - #End if (dimensions check and pressure levels) - #End for (case loop) - #End for (obs/baseline loop) - #End for (variable loop) + # Calculate seasonal means and differences + use_odata = odata_level if has_lev else odata + mseason, oseason, dseason, pseason = process_seasonal_data( + mdata, + use_odata, + plot['season'], vres + ) + + # Create plot + if plot['path'].exists(): + plot['path'].unlink() + + pf.make_polar_plot( + plot['path'], test_nicknames[case_idx], base_nickname, + [syear_cases[case_idx], eyear_cases[case_idx]], + [syear_baseline, eyear_baseline], + mseason, oseason, dseason, pseason, + hemisphere=get_hemisphere(plot['type']), + obs=adfobj.compare_obs, **vres + ) + + # Add to website + adfobj.add_website_data( + plot['path'], plot['var'], case_name, + category=web_category, season=plot['season'], + plot_type=plot['type'] + ) - #Notify user that script has ended: print(" ...polar maps have been generated successfully.") ############## From 5809cdbc7651f7f1b2c13350206640ad5f9c524c Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 7 Mar 2025 09:36:59 -0700 Subject: [PATCH 09/91] remove extra print statements --- scripts/plotting/polar_map.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/plotting/polar_map.py b/scripts/plotting/polar_map.py index 14fb9be1f..a8f5922b5 100644 --- a/scripts/plotting/polar_map.py +++ b/scripts/plotting/polar_map.py @@ -217,13 +217,11 @@ def polar_map(adfobj): has_lev = False if has_lev and pres_levs and plot.get('pressure'): - print("PRESSURE") if not all(dim in mdata.dims for dim in ['lat', 'lev']): continue mdata = mdata.sel(lev=plot['pressure']) odata_level = odata.sel(lev=plot['pressure']) else: - print(f"EXPECT 2D VARIABLE: {odata.shape}") if not pf.lat_lon_validate_dims(mdata): continue From a9f9926506faf842ee0364d40583c2ac6ef7b356 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 7 Mar 2025 09:46:19 -0700 Subject: [PATCH 10/91] remove non-functional code --- scripts/plotting/polar_map.py | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/scripts/plotting/polar_map.py b/scripts/plotting/polar_map.py index a8f5922b5..44721d86a 100644 --- a/scripts/plotting/polar_map.py +++ b/scripts/plotting/polar_map.py @@ -34,9 +34,6 @@ def polar_map(adfobj): msg = "\n Generating polar maps..." print(f"{msg}\n {'-' * (len(msg)-3)}") - # - # Use ADF api to get all necessary information - # var_list = adfobj.diag_var_list model_rgrid_loc = adfobj.get_basic_info("cam_regrid_loc", required=True) @@ -51,28 +48,13 @@ def polar_map(adfobj): syear_cases = adfobj.climo_yrs["syears"] eyear_cases = adfobj.climo_yrs["eyears"] - # CAUTION: - # "data" here refers to either obs or a baseline simulation, - # Until those are both treated the same (via intake-esm or similar) - # we will do a simple check and switch options as needed: + # if doing comparison to obs, but no observations are found, quit if adfobj.get_basic_info("compare_obs"): - #Set obs call for observation details for plot titles - obs = True - - #Extract variable-obs dictionary: var_obs_dict = adfobj.var_obs_dict - - #If dictionary is empty, then there are no observations to regrid to, - #so quit here: if not var_obs_dict: print("\t No observations found to plot against, so no polar maps will be generated.") return - else: - obs = False - data_name = adfobj.get_baseline_info("cam_case_name", required=True) # does not get used, is just here as a placemarker - data_list = [data_name] # gets used as just the name to search for climo files HAS TO BE LIST - data_loc = model_rgrid_loc #Just use the re-gridded model data path - #End if + #Grab baseline years (which may be empty strings if using Obs): syear_baseline = adfobj.climo_yrs["syear_baseline"] @@ -97,12 +79,6 @@ def polar_map(adfobj): print(f"\t NOTE: redo_plot is set to {redo_plot}") #----------------------------------------- - #Set data path variables: - #----------------------- - mclimo_rg_loc = Path(model_rgrid_loc) - if not adfobj.compare_obs: - dclimo_loc = Path(data_loc) - #----------------------- #Determine if user wants to plot 3-D variables on #pressure levels: From 9251a2b8a9d4dad7310a839e1dc4982ea1c252a2 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 7 Mar 2025 09:48:42 -0700 Subject: [PATCH 11/91] remove another non-functional line --- scripts/plotting/polar_map.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/plotting/polar_map.py b/scripts/plotting/polar_map.py index 44721d86a..5ad9867a7 100644 --- a/scripts/plotting/polar_map.py +++ b/scripts/plotting/polar_map.py @@ -35,7 +35,6 @@ def polar_map(adfobj): print(f"{msg}\n {'-' * (len(msg)-3)}") var_list = adfobj.diag_var_list - model_rgrid_loc = adfobj.get_basic_info("cam_regrid_loc", required=True) #Special ADF variable which contains the output paths for #all generated plots and tables for each case: From 27512bdfd0f53557ef5778a22f5e2eb3331594af Mon Sep 17 00:00:00 2001 From: shawnusaf <84995386+shawnusaf@users.noreply.github.com> Date: Wed, 12 Mar 2025 11:09:51 -0400 Subject: [PATCH 12/91] Add files via upload Adds the MOPITT plotting script that can be called from the config yaml file. --- scripts/plotting/MOPITT.py | 504 +++++++++++++++++++++++++++++++++++++ 1 file changed, 504 insertions(+) create mode 100644 scripts/plotting/MOPITT.py diff --git a/scripts/plotting/MOPITT.py b/scripts/plotting/MOPITT.py new file mode 100644 index 000000000..4a12c91ce --- /dev/null +++ b/scripts/plotting/MOPITT.py @@ -0,0 +1,504 @@ +#!/usr/bin/env python +# coding: utf-8 +#CESM and MOPITT monthly comparisons with maps --- 2002-2021 + +# loading +import h5py # For loading he5 files +import glob +# Processing +import pandas as pd +import xarray as xr +import numpy as np +from scipy import interpolate # for vertical interpolation +import time # for timing code +#plotting +import matplotlib.pyplot as plt +import cartopy.crs as ccrs # For plotting maps +import cartopy.feature as cfeature # For plotting maps +from cartopy.util import add_cyclic_point # For plotting maps +import datetime +import os + + +def load_and_extract_grid_hdf(filename,varname): + he5_load = h5py.File(filename, mode='r') + lat = he5_load["/HDFEOS/GRIDS/MOP03/Data Fields/Latitude"][:] + lon = he5_load["/HDFEOS/GRIDS/MOP03/Data Fields/Longitude"][:] + alt = he5_load["/HDFEOS/GRIDS/MOP03/Data Fields/Pressure2"][:] + alt_short = he5_load["/HDFEOS/GRIDS/MOP03/Data Fields/Pressure"][:] + + #LAT-LON variables + if varname=='column': + data_loaded = he5_load["/HDFEOS/GRIDS/MOP03/Data Fields/RetrievedCOTotalColumnDay"][:] + elif varname=='apriori_col': + data_loaded = he5_load["/HDFEOS/GRIDS/MOP03/Data Fields/APrioriCOTotalColumnDay"][:] + elif varname=='apriori_surf': + data_loaded = he5_load["/HDFEOS/GRIDS/MOP03/Data Fields/APrioriCOSurfaceMixingRatioDay"][:] + elif varname=='pressure_surf': + data_loaded = he5_load["/HDFEOS/GRIDS/MOP03/Data Fields/SurfacePressureDay"][:] + #LAT-LON-ALT variables + elif varname=='ak_col': + data_loaded = he5_load["/HDFEOS/GRIDS/MOP03/Data Fields/TotalColumnAveragingKernelDay"][:] + elif varname=='apriori_prof': + data_loaded = he5_load["/HDFEOS/GRIDS/MOP03/Data Fields/APrioriCOMixingRatioProfileDay"][:] + + # create xarray DataArray + if (varname=='column' or varname=='apriori_col' + or varname=='apriori_surf'or varname=='pressure_surf'): + dataset_new = xr.DataArray(data_loaded, dims=["lon","lat"], coords=[lon,lat]) + elif (varname=='ak_col'): + dataset_new = xr.DataArray(data_loaded, dims=["lon","lat","alt"], coords=[lon,lat,alt]) + elif (varname=='apriori_prof'): + dataset_new = xr.DataArray(data_loaded, dims=["lon","lat","alt"], coords=[lon,lat,alt_short]) + + # missing value -> nan + ds_masked = dataset_new.where(dataset_new != -9999.) + he5_load.close() + + return ds_masked + +def collect_mopitt_data(files, varname): + count = 0 + for filename in files: + data = load_and_extract_grid_hdf(filename, varname) + if count == 0: + data_array = data + count += 1 + else: + data_array = xr.concat([data_array, data], 'time') + + return data_array + +def vertical_regrid(input_press, input_values, output_press): + ''' Dimensions must be: ''' + + regrid_array = xr.full_like(output_press, np.nan) + for t in range (input_press.shape[0]): + print(t) + # Latitude values + for y in range (input_press.shape[2]): + # Longitude values + for x in range (input_press.shape[3]): + xx = input_press.isel(time=t, lat=y, lon=x) + yy = input_values.isel(time=t, lat=y, lon=x) + f = interpolate.interp1d(xx, yy, fill_value="extrapolate") + xnew = output_press.isel(time=t, lat=y, lon=x) + regrid_array[t, x, y, :] = f(xnew) + return regrid_array + +def MOPITT(adfobj): + + # ### Load Measurements + # MOPITT CO V9 daytime only joint product, L3 gridded 1x1. + # Climatology data archived through NCAR GDEX, Data Reference: https://gdex.ucar.edu/dataset/369_buchholz.html + + files = sorted(glob.glob('/glade/campaign/acom/acom-da/buchholz/MOPITT_v9_climo/*2002_2021.he5', recursive=False)) + + data_array = collect_mopitt_data(files, "column") + + sd_files = sorted(glob.glob('/glade/campaign/acom/acom-da/buchholz/MOPITT_v9_climo/*2002_2021_SD.he5', recursive=False)) + + # Latitude and lanogitude not loading correctly + sd_array = collect_mopitt_data(sd_files, "column") + #Replace coordinate variables (because SD of coordinate variables was saved and that is zero) + sd_array = sd_array.assign_coords(lon=data_array.lon, lat=data_array.lat) + + #surface pressure + sat_psurf = collect_mopitt_data(files, "pressure_surf") + + # load column and surface a priori + prior_col_array = collect_mopitt_data(files, "apriori_col") + prior_surf = collect_mopitt_data(files, "apriori_surf") + + # load profile a priori + prior_prof_array = collect_mopitt_data(files, "apriori_prof") + + # averaging kernel + ak_array = collect_mopitt_data(files, "ak_col") + + # broadcast 9 levels 900 to 100 hPa repeated everywhere + dummy, press_dummy_arr = xr.broadcast(prior_prof_array,prior_prof_array.alt) + + # create array to hold 9 regular spaced, plus floating surface pressure + sat_pressure_array = xr.full_like(ak_array, np.nan) + sat_pressure_array[:,:,:,0] = sat_psurf + sat_pressure_array[:,:,:,1:10] = press_dummy_arr + #print(sat_pressure_array.isel(time=0, lat=-45, lon=-80)) + + #Correct for where MOPITT surface pressure <900 hPa + #calculate pressure differences + dp = xr.full_like(ak_array, np.nan) + dp.shape + dp[:,:,:,9] = 1000 + for z in range (0, 9): + dp[:,:,:,z] = sat_pressure_array[:,:,:,0] - sat_pressure_array[:,:,:,z+1] + + #Repeat surface values at all levels to replace in equivalent position in parray if needed + psurfarray = xr.full_like(ak_array, np.nan) + for z in range (0, 10): + psurfarray[:,:,:,z] = sat_psurf + + #Add fill values below true surface + new_pressure_array = sat_pressure_array.copy() + new_pressure_array = new_pressure_array.where(dp>0) + #replace lowest pressure with surface pressure + new_pressure_array = psurfarray.where((dp>0) & (dp<100),new_pressure_array) + + # Model layer values are averages for the whole box, centred at an altitude, + # while MOPITT values are averages described for the whole box above level. + # Therefore need to interp MOPITT pressures to mid-box locations + pinterp = xr.full_like(ak_array, np.nan) + pinterp[:,:,:,9] = 87. + for z in range (0, 9): + pinterp[:,:,:,z] = new_pressure_array[:,:,:,z] - (new_pressure_array[:,:,:,z]-new_pressure_array[:,:,:,z+1])/2 + + # MOPITT surface values are stored separately to profile values because of the floating surface pressure. + # So, for calculations, need to combine + # Repeat surface a priori values at all levels to replace if needed + apsurfarray = xr.full_like(ak_array, np.nan) + for z in range (0, 10): + apsurfarray[:,:,:,z] = prior_surf + + aparray = xr.full_like(ak_array, np.nan) + aparray[:,:,:,0] = prior_surf + aparray[:,:,:,1:10] = prior_prof_array + aparray = aparray.where(dp>0) + aparray = apsurfarray.where((dp>0) & (dp<100),aparray) + + # ### Define the directories and file of interest for your model results. + + ntest = len(adfobj.get_cam_info('cam_case_name', required=True)) + redo_plot = adfobj.get_basic_info('redo_plot') + print(f"\t NOTE: redo_plot is set to {redo_plot}") + + for i in range(0,ntest): + + test_case = adfobj.get_cam_info('cam_case_name', required=True)[i] + cam_climo_loc = adfobj.get_cam_info('cam_climo_loc',required=True) + cam_hist_loc = adfobj.get_cam_info('cam_hist_loc',required=True)[i] + test_nicknames = adfobj.case_nicknames["test_nicknames"][i] + plot_locations = adfobj.plot_location[i] + + # Will need to calculate and load a climatology of output for ADF processing + result_dir = cam_hist_loc + + hist_str = adfobj.get_basic_info('hist_str') + hfiles = result_dir+"/*cam.h0a.*" + + #the netcdf file is now held in an xarray dataset named 'nc_load' and can be referenced later in the notebook + nc_load = xr.open_mfdataset(hfiles,combine='nested',concat_dim='time') + nc_load=nc_load.groupby('time.month').mean(dim='time') + nc_load['month']=[datetime.datetime(2003,month,1) for month in nc_load['month'].values] + nc_load=nc_load.rename({'month': 'time'}) + #to see what the netCDF file contains, just call the variable you read it into nc_load + + # Convert longitudes to -180 to 180 + nc_load = nc_load.assign_coords(lon=(((nc_load.lon + 180) % 360) - 180)).sortby('lon') + + #extract variable + var_sel = nc_load['CO'].load()/1e-09 + + # Model pressure values + # model pressure is in Pa + psurf = nc_load['PS'].load() + pdel = nc_load['PDELDRY'].load()/100 + + # Load values to create pressure array id PDELDRY was not saved + # using hybrid coordinate variable definitons + # https://www2.cesm.ucar.edu/models/atm-cam/docs/usersguide/node25.html + # model pressure is in Pa + # interfaces (edges) + hyai = nc_load['hyai'].load() + hybi = nc_load['hybi'].load() + p0 = 100000.0 + lev = var_sel.coords['lev'] + num_lev = lev.shape[0] + + # Initialize pressure edge arrays + mod_press_low = xr.zeros_like(var_sel) + mod_press_top = xr.zeros_like(var_sel) + + # Calculate pressure edge arrays + # CAM-chem layer indices start at the top and end at the bottom + for j in range(num_lev): + mod_press_top[:,j,:,:] = hyai[:,j]*p0 + hybi[:,j]*psurf + mod_press_low[:,j,:,:] = hyai[:,j+1]*p0 + hybi[:,j+1]*psurf + + # Delta P in hPa + mod_deltap = (mod_press_low - mod_press_top)/100 + # CHECK -----> + # Calculated in python should be within 3 to 4 decimal places of model calculated PDELDRY + + # Pressure mid-layer values + mod_press_mid = (mod_press_top + mod_press_low)/200 + + dates = var_sel.coords['time'] + + # ### Regrid model to MOPITT horizontal grid + # Horizontal regridding can have many options. Currently using xarray interp + + # linear interp + tracer_regrid = var_sel.interp(coords=dict(lat=data_array.lat, lon=data_array.lon), method='linear') + mod_deltap_regrid = pdel.interp(coords=dict(lat=data_array.lat, lon=data_array.lon), method='linear') + psurf_regrid = psurf.interp(coords=dict(lat=data_array.lat, lon=data_array.lon), method='linear') + model_pressure_regrid = mod_press_mid.interp(coords=dict(lat=data_array.lat, lon=data_array.lon), method='linear') + + # ### Vertical interpolate model grid to MOPITT vertical layers + + start = time.perf_counter() + model_vert_regrid = vertical_regrid(model_pressure_regrid, tracer_regrid, pinterp) + end = time.perf_counter() + print('This took '+ str((end-start)/60) + ' minutes to run') + + # ### Consistency check: plot profile + # Need to check the interpolation and AK application is doing what I think it is doing. + + def profile_plot(x,y,color_choice,label_string,linewidth,marker): + plt.plot(x, y, marker, label=label_string, + color=color_choice, + markersize=10, linewidth=linewidth, + markerfacecolor=color_choice, + markeredgecolor='grey', + markeredgewidth=1) + + plt.figure(figsize=(8,10)) + ax = plt.axes() + ax.invert_yaxis() + + #-------------------| variable |------------------------| pressure |--------- + profile_plot(aparray.isel(time=0, lat=45, lon=105), pinterp.isel(time=0, lat=45, lon=105), 'blue','MOPITT a priori',8,'-ok') + profile_plot(var_sel.isel(time=0, lat=45, lon=105), mod_press_mid.isel(time=0, lat=45, lon=105), 'red','CAM-chem',8,'-ok') + profile_plot(tracer_regrid.isel(time=0, lat=45, lon=105), model_pressure_regrid.isel(time=0, lat=45, lon=105), 'green','CAM-chem horizontal regrid',8,'-ok') + profile_plot(model_vert_regrid.isel(time=0, lat=45, lon=105), pinterp.isel(time=0, lat=45, lon=105), 'gold','CAM-chem horizontal and vertical regrid',2,'-ok') + + #titles + plt.title('Profile example at ',fontsize=24) + plt.xlabel('VMR',fontsize=24) + plt.ylabel('Pressure',fontsize=24) + + # legend + plt.legend(bbox_to_anchor=(1.9, 0.78),loc='lower right',fontsize=18) + + plt.show() + + # ### Convert to total column for comparison + + # Convert base model and regridded model values to total column amounts in case you need for comparison. This step is not necessary for the final plots, but a good consistency check if needed at a later stage. + + #------------------------------- + #CONSTANTS and conversion factor + #------------------------------- + NAv = 6.0221415e+23 #--- Avogadro's number + g = 9.81 #--- m/s - gravity + MWair = 28.94 #--- g/mol + xp_const = (NAv* 10)/(MWair*g)*1e-09 #--- scaling factor for turning vmr into pcol + #--- (note 1*e-09 because in ppb) + + var_tcol = xr.dot(pdel, xp_const*var_sel, dims=["lev"]) + var_tcol_regrid = xr.dot(mod_deltap_regrid, xp_const*tracer_regrid, dims=["lev"]) + + # ### Apply MOPITT AK + # Smooth model data to measurement space according to the User Guide: + # https://www2.acom.ucar.edu/sites/default/files/documents/v9_users_guide_20220203.pdf + # + # Csmooth = Cprior + AK(xmodel−xprior) + # + # C = column + # x = profile + # + # Note MOPITT AKs are applied to log10(VMR) + # + + #from math import log10 + log_ap = np.log10(aparray) + log_model=np.log10(model_vert_regrid) + + diff_array = log_model-log_ap + + ak_appl = ak_array * diff_array + smoothed_model = prior_col_array + np.sum(ak_appl, axis=3) + + # ### Difference + # The model - observations array + + tcol_diff = smoothed_model - data_array + + # ### Plot the map comparisons + + # Add cyclic point to avoid white line over Africa + lon_model = var_sel.lon + + var_tcol_cyc, lon_cyc = add_cyclic_point(var_tcol, coord=lon_model) + + month_titles = np.array(['January', 'February', 'March', 'April', 'May', 'June', + 'July','August','September','October','November','December']) + + #Sub-plots function + def map_subplot(lon,lat,var,contours,colormap,labelbar,position1,position2): + axs[position1,position2].contourf(lon,lat,var,contours,cmap=colormap,extend=labelbar) + axs[position1,position2].coastlines() + #gridlines + gl = axs[position1,position2].gridlines(crs=ccrs.PlateCarree(), draw_labels=True, color='grey', + linewidth=2, alpha=0.5, linestyle='--') + gl.top_labels = False + gl.right_labels = False + gl.xlabel_style = {'size': 42, 'color': 'gray'} + gl.ylabel_style = {'size': 42, 'color': 'gray'} + #latitude limits in degrees + axs[position1,position2].set_ylim(-70,70) + # add coastlines + axs[position1,position2].add_feature(cfeature.COASTLINE) + axs[position1,position2].add_feature(cfeature.LAKES) + axs[position1,position2].add_feature(cfeature.BORDERS) + + #Create monthly plot + for monthval in range(0,12): + + #MONTH definiton for plotting (0 to 11): + + oCompare_Plot=plot_locations+'/MOPITT_'+month_titles[monthval]+'_ANN_Special_Mean.png' + + if (not(redo_plot)) and (os.path.isfile(oCompare_Plot)): + print(month_titles[monthval],' monthly plot exists and redo_plot is false. Adding to website and Skipping plot.') + adfobj.add_website_data(oCompare_Plot,"MOPITT_"+month_titles[monthval], None,season="ANN", multi_case=True,category="MOPITT_DIAGNOSTICS") + continue + else: + print("Plotting month ",month_titles[monthval]) + + #-----------------columns, rows + fig, axs = plt.subplots(2,2,figsize=(50,21), + subplot_kw={'projection': ccrs.PlateCarree()}, + constrained_layout=True) + + fig.set_constrained_layout_pads(w_pad=4 / 72, h_pad=4 / 72, hspace=0, wspace=0) + fig.suptitle('Total column carbon monoxide (CO): '+month_titles[monthval], fontsize=56) + + #------------------ Create the plots + #define contour levels + clev = np.arange(0.4, 3.6, 0.1) + + # Plot MOPITT columns + x = 0 + y = 0 + map_subplot(data_array.lon,data_array.lat,data_array[monthval,:,:].transpose()/1e18,clev,'Spectral_r','both',x,y) + axs[x,y].set_title('MOPITT CO 2002-2021', fontsize=56) + + # Plot MOPITT CO standard deviation + x = 0 + y = 1 + map_subplot(data_array.lon,data_array.lat,sd_array[monthval,:,:].transpose()/1e17,clev,'Spectral_r','both',x,y) + axs[x,y].set_title('MOPITT CO standard deviation', fontsize=56) + + # Plot CAM-chem columns smooth + x = 1 + y = 0 + map_subplot(smoothed_model.lon,smoothed_model.lat,smoothed_model[monthval,:,:].transpose()/1e18,clev,'Spectral_r','both',x,y) + axs[x,y].set_title('Model CO (smoothed)', fontsize=56) + + # Plot CAM-chem -- MOPITT diff + #new contour levels + clevII = np.arange(-3.5, 3.6, 0.1) + x = 1 + y = 1 + map_subplot(tcol_diff.lon,tcol_diff.lat,tcol_diff[monthval,:,:].transpose()/1e18,clevII,'bwr','both',x,y) + axs[x,y].set_title('Model - Obs difference', fontsize=56) + + #------------------ Axes titles + # x-axis + for j in range(2): + axs[1,j].text(0.50, -0.25, 'Longitude', va='bottom', ha='center', + rotation='horizontal', rotation_mode='anchor', + transform=axs[1,j].transAxes, fontsize=56) + + # y-axis + for j in range(2): + axs[j,0].text(-0.15, 0.52, 'Latitude', va='bottom', ha='center', + rotation='vertical', rotation_mode='anchor', + transform=axs[j,0].transAxes, fontsize=56) + + #------------------ Colorbar definitions + # defining these colorbars re-plots the plots and can take some time... It would be great to find a better way to do this + # column + cbar_set = axs[0,0].contourf(data_array.lon,data_array.lat,data_array[monthval,:,:].transpose()/1e18,clev,cmap='Spectral_r',extend='both') + cb = fig.colorbar(cbar_set, ax=axs[0, 0], shrink=0.95) + cb.set_label(label='CO (x $10^{18}$ molec./cm$^2$)', fontsize=22) + cb.ax.tick_params(labelsize=22) + cbar_set = axs[1,0].contourf(smoothed_model.lon,smoothed_model.lat,smoothed_model[monthval,:,:].transpose()/1e18,clev,cmap='Spectral_r',extend='both') + cb = fig.colorbar(cbar_set, ax=axs[1, 0], shrink=0.95) + cb.set_label(label='CO (x $10^{18}$ molec./cm$^2$)', fontsize=22) + cb.ax.tick_params(labelsize=22) + # column SD + cbar_set = axs[0,1].contourf(sd_array.lon,sd_array.lat,sd_array[monthval,:,:].transpose()/1e17,clev,cmap='Spectral_r',extend='both') + cb = fig.colorbar(cbar_set, ax=axs[0, 1], shrink=0.95) + cb.set_label(label='CO (x $10^{17}$ molec./cm$^2$)', fontsize=22) + cb.ax.tick_params(labelsize=22) + # difference + cbarII_set = axs[1,1].contourf(tcol_diff.lon,tcol_diff.lat,tcol_diff[monthval,:,:].transpose()/1e18,clevII,cmap='bwr',extend='both') + cbII = fig.colorbar(cbarII_set, ax=axs[1, 1], shrink=0.95) + cbII.set_label(label='CO difference (x $10^{18}$ molec./cm$^2$)', fontsize=22) + cbII.ax.tick_params(labelsize=22) + + plt.savefig(oCompare_Plot) + print(oCompare_Plot) + adfobj.add_website_data(oCompare_Plot,"MOPITT_"+month_titles[monthval], None,season="ANN", multi_case=True,category="MOPITT_DIAGNOSTICS") + + # ### Seasonal plot of differences + + # Create seasonal averages + oCompare_Season=plot_locations+'/MOPITT_SEASONAL_ANN_Special_Mean.png' + if (not(redo_plot)) and (os.path.isfile(oCompare_Season)): + print(month_titles[monthval],' seasonal plot exists and redo_plot is false. Adding to website and Skipping plot.') + adfobj.add_website_data(oCompare_Season,'MOPITT_SEASONAL', None, season="ANN", multi_case=True,category="MOPITT_DIAGNOSTICS") + continue + else: + print("Plotting seasonal") + + seas_diff = xr.full_like(tcol_diff.isel(time=[0,1,2,3]), np.nan) + seas_diff[0,:,:] = tcol_diff.isel(time=[0,1,11]).mean('time') + seas_diff[1,:,:] = tcol_diff.isel(time=[2,3,4]).mean('time') + seas_diff[2,:,:] = tcol_diff.isel(time=[5,6,7]).mean('time') + seas_diff[3,:,:] = tcol_diff.isel(time=[8,9,10]).mean('time') + seas_diff.shape + + titles = np.array([['(a) DJF','(b) MAM'],['(c) JJA','(d) SON']]) + + fig, axs = plt.subplots(2,2,figsize=(50,21), + subplot_kw={'projection': ccrs.PlateCarree()}, + constrained_layout=True) + + fig.set_constrained_layout_pads(w_pad=4 / 72, h_pad=4 / 72, hspace=0, wspace=0) + + fig.suptitle('Total column carbon monoxide difference: Model - MOPITT', fontsize=56) + + #define contour levels + clev = np.arange(-3.5, 3.6, 0.1) + + for x in range(2): + for y in range(2): + season_index = x*2 + y + map_subplot(seas_diff.lon,seas_diff.lat,seas_diff[season_index,:,:].transpose()/1e18,clev,'bwr','both',x,y) + #sub-titles + axs[x,y].set_title(titles[x,y], fontsize=48) + + #------------------ Axes titles + # x-axis + for j in range(2): + axs[1,j].text(0.50, -0.25, 'Longitude', va='bottom', ha='center', + rotation='horizontal', rotation_mode='anchor', + transform=axs[1,j].transAxes, fontsize=56) + + # y-axis + for j in range(2): + axs[j,0].text(-0.15, 0.52, 'Latitude', va='bottom', ha='center', + rotation='vertical', rotation_mode='anchor', + transform=axs[j,0].transAxes, fontsize=56) + + #------------------ Colorbar definitions + # note the hard-coded value here... this overplots the top left plot + cbar_set = axs[0,0].contourf(seas_diff.lon,seas_diff.lat,seas_diff[0,:,:].transpose()/1e18,clev,cmap='bwr',extend='both') + cb = fig.colorbar(cbar_set, ax=axs[:, 1], shrink=0.7) + cb.set_label(label='CO (x $10^{18}$ molec./cm$^2$)', fontsize=42) + cb.ax.tick_params(labelsize=42) + + plt.savefig(oCompare_Season) + adfobj.add_website_data(oCompare_Season,'MOPITT_SEASONAL', None, season='ANN', multi_case=True,category="MOPITT_DIAGNOSTICS") From d4c3c843af66d97557caacd86dcb74f68cca6af5 Mon Sep 17 00:00:00 2001 From: shawnusaf <84995386+shawnusaf@users.noreply.github.com> Date: Wed, 12 Mar 2025 11:11:32 -0400 Subject: [PATCH 13/91] Update config_cam_baseline_example.yaml Adds the MOPITT entry in the 'plotting_scripts' section of the cam config file, and adds CO to the list of cam variables to plot. --- config_cam_baseline_example.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config_cam_baseline_example.yaml b/config_cam_baseline_example.yaml index d9f3ad4c2..89aa4ec5a 100644 --- a/config_cam_baseline_example.yaml +++ b/config_cam_baseline_example.yaml @@ -481,6 +481,7 @@ plotting_scripts: - cam_taylor_diagram - qbo - ozone_diagnostics + - MOPITT #- tape_recorder #- tem #- regional_map_multicase #To use this please un-comment and fill-out @@ -506,6 +507,7 @@ diag_var_list: - FLNT - LANDFRAC - O3 + - CO # # MDTF recommended variables From 42a16fbcac15ea8b26a8c7c9ca27ccba392cb88c Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 13 Mar 2025 16:52:15 -0600 Subject: [PATCH 14/91] check for levels in the variable earlier, improve logic for pressure level plots --- scripts/plotting/polar_map.py | 47 ++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/scripts/plotting/polar_map.py b/scripts/plotting/polar_map.py index 5ad9867a7..bd9540c29 100644 --- a/scripts/plotting/polar_map.py +++ b/scripts/plotting/polar_map.py @@ -117,29 +117,36 @@ def polar_map(adfobj): for case_idx, case_name in enumerate(case_names): plot_loc = Path(plot_locations[case_idx]) - + + tmp_ds = adfobj.data.load_regrid_dataset(case_name, var) + if tmp_ds is None: + continue + + has_lev = "lev" in tmp_ds.dims + for s in seasons: for hemi_type in ["NHPolar", "SHPolar"]: if pres_levs: - for pres in pres_levs: - plot_name = plot_loc / f"{var}_{pres}hpa_{s}_{hemi_type}_Mean.{plot_type}" - info = { - 'path': plot_name, - 'var': f"{var}_{pres}hpa", - 'case': case_name, - 'case_idx': case_idx, - 'season': s, - 'type': hemi_type, - 'pressure': pres, - 'exists': plot_name.is_file() - } - plot_info.append(info) - if not (redo_plot or not info['exists']): - adfobj.add_website_data(info['path'], info['var'], - info['case'], category=web_category, - season=s, plot_type=hemi_type) - else: - all_plots_exist = False + if has_lev: + for pres in pres_levs: + plot_name = plot_loc / f"{var}_{pres}hpa_{s}_{hemi_type}_Mean.{plot_type}" + info = { + 'path': plot_name, + 'var': f"{var}_{pres}hpa", + 'case': case_name, + 'case_idx': case_idx, + 'season': s, + 'type': hemi_type, + 'pressure': pres, + 'exists': plot_name.is_file() + } + plot_info.append(info) + if not (redo_plot or not info['exists']): + adfobj.add_website_data(info['path'], info['var'], + info['case'], category=web_category, + season=s, plot_type=hemi_type) + else: + all_plots_exist = False else: plot_name = plot_loc / f"{var}_{s}_{hemi_type}_Mean.{plot_type}" info = { From fb7f9050ce5865fb47c369e85e9df63feac85cec Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 14 Mar 2025 14:40:43 -0600 Subject: [PATCH 15/91] refactored global_latlon_map.py; split aod special case into module --- scripts/plotting/aod_latlon.py | 483 ++++++++++++ scripts/plotting/global_latlon_map.py | 1017 +++++++------------------ 2 files changed, 742 insertions(+), 758 deletions(-) create mode 100644 scripts/plotting/aod_latlon.py diff --git a/scripts/plotting/aod_latlon.py b/scripts/plotting/aod_latlon.py new file mode 100644 index 000000000..1c8c59bf8 --- /dev/null +++ b/scripts/plotting/aod_latlon.py @@ -0,0 +1,483 @@ +"""Module for AOD-specific plotting functionality""" + +from pathlib import Path +import numpy as np +import xarray as xr +import xesmf as xe + + +import matplotlib as mpl +import matplotlib.pyplot as plt +from matplotlib import gridspec +import cartopy.crs as ccrs +from cartopy.mpl.ticker import LongitudeFormatter, LatitudeFormatter +from cartopy.util import add_cyclic_point + +import plotting_functions as pf + +from dataclasses import dataclass + +@dataclass +class AODPlotConfig: + """Configuration for AOD plots.""" + seasons: list = ('DJF', 'MAM', 'JJA', 'SON') + season_names: dict = {'DJF': 'Dec-Jan-Feb', 'MAM': 'Mar-Apr-May', 'JJA': 'Jun-Jul-Aug', 'SON': 'Sep-Oct-Nov'} + obs_sources: list = ('TERRA MODIS', 'MERRA2') + var_name: str = 'AODVISdn' + + +def aod_latlon(adfobj): + """Generate AOD comparison plots.""" + config = AODPlotConfig() + + # Load observations + obs_data = load_observations(adfobj) + if not obs_data: + return + + # Process model data + model_data = process_model_cases(adfobj, config.var_name, obs_data) + if not model_data: + return + + # Generate plots + for obs_source, obs_dataset in obs_data.items(): + for season in config.seasons: + create_aod_panel(adfobj, model_data, obs_dataset, + season, obs_source) + + +def load_observations(adfobj): + """Load MERRA2 and MODIS observation datasets. + + Parameters + ---------- + adfobj : AdfDiag + The diagnostics object containing configuration + + Returns + ------- + dict + Dictionary of observation datasets keyed by source name + """ + obs_dir = adfobj.get_basic_info("obs_data_loc") + obs_files = { + 'TERRA MODIS': 'MOD08_M3_192x288_AOD_2001-2020_climo.nc', + 'MERRA2': 'MERRA2_192x288_AOD_2001-2020_climo.nc' + } + + obs_data = {} + for source, filename in obs_files.items(): + ds = load_obs_data(obs_dir, filename) + if ds is None: + print(f"\t WARNING: AOD Panel plots not made, missing {source} file") + return None + + # Extract correct variable based on source + if source == 'MERRA2': + ds = ds['TOTEXTTAU'] + else: # MODIS + ds = ds['AOD_550_Dark_Target_Deep_Blue_Combined_Mean_Mean'] + + # Calculate seasonal means + ds_seasonal = monthly_to_seasonal(ds, obs=True) + obs_data[source] = ds_seasonal + + return obs_data + + +def load_obs_data(obs_dir, file_name): + """Load and prepare observational dataset.""" + file_path = Path(obs_dir) / file_name + if not file_path.is_file(): + return None + + ds = xr.open_dataset(file_path) + # Round coordinates for consistency + ds['lon'] = ds['lon'].round(5) + ds['lat'] = ds['lat'].round(5) + return ds + + +def process_model_cases(adfobj, var, obs_data): + """Process model cases and regrid if necessary. + + Parameters + ---------- + adfobj : AdfDiag + The diagnostics object containing configuration + var : str + Variable name to process + obs_data : dict + Dictionary of observation datasets with their grids + + Returns + ------- + list + List of processed model datasets, one per case + """ + # Get case information + cases = adfobj.get_cam_info('cam_case_name', required=True) + if not adfobj.compare_obs: + cases = cases + [adfobj.data.ref_case_label] # ref case added to cases + + # Get reference grid from first observation dataset + ref_obs = next(iter(obs_data.values())) + + # Process each case + processed_data = [] + for case_name in cases: + # Load and process model data + case_data = process_model_data(adfobj, case_name, var, ref_obs) + if case_data is not None: + processed_data.append((case_data, case_name)) + + return processed_data if processed_data else None + + +def process_model_data(adfobj, case_name, var, obs_shape): + """Process model data and check grid compatibility.""" + ds_case = adfobj.data.load_climo_da(case_name, var) + if ds_case is None: + print(f"\t WARNING: No climo file for {case_name} variable {var}") + return None + + ds_case['lon'] = ds_case['lon'].round(5) + ds_case['lat'] = ds_case['lat'].round(5) + + # Check grid compatibility + needs_regrid = check_grid_compatibility(ds_case, obs_shape) + if needs_regrid: + ds_case = regrid_to_obs(ds_case, obs_shape) + + return monthly_to_seasonal(ds_case) + + +def check_grid_compatibility(model_arr, obs_arr): + """Check if model grid matches observation grid. + + Parameters + ---------- + model_arr : xarray.DataArray + Model data array with lat/lon coordinates + obs_arr : xarray.DataArray + Observation data array with lat/lon coordinates + + Returns + ------- + bool + True if grids don't match and regridding is needed + """ + test_lons = model_arr.lon + test_lats = model_arr.lat + obs_lons = obs_arr.lon + obs_lats = obs_arr.lat + + # Check if shapes match first + if obs_lons.shape != test_lons.shape: + return True + + # Check exact coordinate matches + try: + xr.testing.assert_equal(test_lons, obs_lons) + xr.testing.assert_equal(test_lats, obs_lats) + return False + except AssertionError: + return True + +def create_aod_panel(adfobj, data_sets, obs_dataset, season, obs_name): + """Create AOD panel plot with differences and percent differences.""" + plot_data = [] + plot_titles = [] + plot_params = [] + case_names = [] + types = [] + + # Get plot parameters from configuration + plot_config = get_plot_params(adfobj) + + for case_data, case_name in data_sets: + # Calculate differences + diff = calculate_differences(case_data, obs_dataset, season) + plot_data.append(diff) + plot_titles.append(make_plot_config(diff, case_name, obs_name, season, "Diff")) + plot_params.append(plot_config['default']) + case_names.append(case_name) + types.append("Diff") + + # Calculate percent differences + pdiff = calculate_percent_diff(case_data, obs_dataset, season) + plot_data.append(pdiff) + plot_titles.append(make_plot_config(pdiff, case_name, obs_name, season, "Percent Diff")) + plot_params.append(plot_config['relerr']) + case_names.append(case_name) + types.append("Percent Diff") + + return aod_panel_latlon(adfobj, plot_titles, plot_params, plot_data, + season, obs_name, case_names, len(data_sets), + types, symmetric=True) + + +def validate_obs_data(merra_data, modis_data): + """Validate observation datasets.""" + if merra_data is None or modis_data is None: + raise ValueError("Missing observation data") + + if not np.array_equal(merra_data.lat, modis_data.lat): + raise ValueError("Observation grids do not match") + + +def regrid_to_obs(model_arr, obs_arr): + """Regrid model data to match observation grid using bilinear interpolation. + + Parameters + ---------- + model_arr : xarray.DataArray + Model data array to be regridded + obs_arr : xarray.DataArray + Observation data array with target grid + + Returns + ------- + xarray.DataArray + Regridded model data, or None if grids already match + """ + # Create target grid specification + ds_out = xr.Dataset({ + "lat": (["lat"], obs_arr.lat.values, {"units": "degrees_north"}), + "lon": (["lon"], obs_arr.lon.values, {"units": "degrees_east"}) + }) + + # Perform regridding + regridder = xe.Regridder(model_arr, ds_out, "bilinear", periodic=True) + model_regrid = regridder(model_arr, keep_attrs=True) + + return model_regrid + +def calculate_differences(case_data, obs_data, season): + """Calculate differences between case and observation data for a given season. + + Parameters + ---------- + case_data : xarray.DataArray + Model case data + obs_data : xarray.DataArray + Observation data + season : str + Season to calculate difference for + + Returns + ------- + xarray.DataArray + Difference between case and observation data + """ + return case_data.sel(season=season) - obs_data.sel(season=season) + + +def calculate_percent_diff(case_data, obs_data, season): + """Calculate percent difference between case and observation data. + + Parameters + ---------- + case_data : xarray.DataArray + Model case data + obs_data : xarray.DataArray + Observation data + season : str + Season to calculate difference for + + Returns + ------- + xarray.DataArray + Percent difference, clipped to [-100, 100] + """ + diff = calculate_differences(case_data, obs_data, season) + pdiff = 100 * diff / obs_data.sel(season=season) + return np.clip(pdiff, -100, 100) + + +def make_plot_config(data, case_name, obs_name, season, plot_type): + """Create plot configuration dictionary. + + Parameters + ---------- + data : xarray.DataArray + Data to plot + case_name : str + Name of case being plotted + obs_name : str + Name of observation dataset + season : str + Season being plotted + plot_type : str + Type of plot ('Diff' or 'Percent Diff') + + Returns + ------- + dict + Plot configuration including data and metadata + """ + config = AODPlotConfig() + return { + 'data': data, + 'title': f'{case_name} - {obs_name}\nAOD 550 nm - {config.season_names[season]}', + 'case_name': case_name, + 'plot_type': plot_type, + 'season': season + } + + +def get_plot_params(adfobj): + """Get AOD plot parameters from ADF configuration.""" + res = adfobj.variable_defaults + res_aod_diags = res.get("aod_diags", {}) + return { + 'default': res_aod_diags.get("plot_params", {}), + 'relerr': res_aod_diags.get("plot_params_relerr", {}) + } + +### refactored aod_panel_latlon: +def aod_panel_latlon(adfobj, plot_titles, plot_params, data, season, obs_name, case_names, case_num, types, symmetric=False): + """Create AOD panel plot with model vs observation differences. + + Parameters + ---------- + adfobj : AdfDiag + The diagnostics object containing configuration + plot_titles : list + List of titles for each panel + plot_params : list + List of plotting parameters for each panel + data : list + List of xarray DataArrays to plot + season : str + Current season being plotted + obs_name : str + Name of observation dataset + case_names : list + List of case names + case_num : int + Number of cases + types : list + List of plot types ('Diff' or 'Percent Diff') + symmetric : bool, optional + Whether to use symmetric colormap, by default False + """ + # Get plot configuration + file_type = adfobj.read_config_var("diag_basic_info").get('plot_type', 'png') + plot_dir = adfobj.plot_location[0] + plotfile = Path(plot_dir) / f'AOD_diff_{obs_name.replace(" ","_")}_{season}_LatLon_Mean.{file_type}' + + # Check if plot should be regenerated + if plotfile.is_file() and not adfobj.get_basic_info('redo_plot'): + adfobj.add_website_data(plotfile, f'AOD_diff_{obs_name.replace(" ","_")}', None, + season=season, multi_case=True, plot_type="LatLon", + category="4-Panel AOD Diags") + return + + # Create figure and axes + fig = plt.figure(figsize=(7*case_num, 10)) + gs = mpl.gridspec.GridSpec(2*case_num, int(3*case_num), wspace=0.5, hspace=0.0) + gs.tight_layout(fig) + + axs = [] + for i in range(case_num): + start = i * 3 + end = (i + 1) * 3 + axs.append(plt.subplot(gs[0:case_num, start:end], projection=ccrs.PlateCarree())) + axs.append(plt.subplot(gs[case_num:, start:end], projection=ccrs.PlateCarree())) + + # Generate each panel + for i, field in enumerate(data): + # Create individual plot + ind_fig, ind_ax = plt.subplots(1, 1, figsize=((7*case_num)/2, 10/2), + subplot_kw={'projection': ccrs.PlateCarree()}) + + # Prepare data + field_values = field.values[:,:] + lon_values = field.lon.values + lat_values = field.lat.values + field_values, lon_values = add_cyclic_point(field_values, coord=lon_values) + lon_mesh, lat_mesh = np.meshgrid(lon_values, lat_values) + field_mean = np.nanmean(field_values) + + # Set plot parameters + plot_param = plot_params[i] + levels = np.linspace(plot_param['range_min'], plot_param['range_max'], + plot_param['nlevel'], endpoint=True) + if 'augment_levels' in plot_param: + levels = sorted(np.append(levels, np.array(plot_param['augment_levels']))) + + plot_config = plot_titles[i] + title = f"{plot_config['title']} Mean {field_mean:.2g}" + + # Create plots + cmap_option = (plot_param.get('colormap', plt.cm.bwr) if symmetric + else plot_param.get('colormap', plt.cm.turbo)) + extend_option = 'both' if symmetric else 'max' + + for ax, is_panel in [(axs[i], True), (ind_ax, False)]: + img = ax.contourf(lon_mesh, lat_mesh, field_values, + levels, cmap=cmap_option, extend=extend_option, + transform=ccrs.PlateCarree()) + ax.set_facecolor('gray') + ax.coastlines() + ax.set_title(title, fontsize=10) + + cbar = plt.colorbar(img, orientation='horizontal', pad=0.05) + if 'ticks' in plot_param: + cbar.set_ticks(plot_param['ticks']) + if 'tick_labels' in plot_param: + cbar.ax.set_xticklabels(plot_param['tick_labels']) + cbar.ax.tick_params(labelsize=6) + + # Save individual plot + pbase = f'AOD_{case_names[i]}_vs_{obs_name.replace(" ","_")}_{types[i].replace(" ","_")}' + ind_plotfile = Path(plot_dir) / f'{pbase}_{season}_LatLon_Mean.{file_type}' + ind_fig.savefig(ind_plotfile, bbox_inches='tight', dpi=300) + plt.close(ind_fig) + + # Save panel plot + fig.savefig(plotfile, bbox_inches='tight', dpi=300) + adfobj.add_website_data(plotfile, f'AOD_diff_{obs_name.replace(" ","_")}', None, + season=season, multi_case=True, plot_type="LatLon", + category="4-Panel AOD Diags") + plt.close(fig) + + +def monthly_to_seasonal(ds, obs=False): + """Convert monthly data to seasonal means. + + Parameters + ---------- + ds : xarray.Dataset or xarray.DataArray + Input data with monthly time dimension + obs : bool, optional + Whether input is observation data, by default False + + Returns + ------- + xarray.DataArray + Data array with new season dimension + """ + seasons = ['DJF', 'MAM', 'JJA', 'SON'] + dataarrays = [] + + if obs and isinstance(ds, xr.Dataset): + # Handle observation dataset with multiple variables + for varname in ds.data_vars: + if '_n' not in varname: # Skip count variables + var_data = ds[varname] + for s in seasons: + dataarrays.append(pf.seasonal_mean(var_data, season=s, is_climo=True)) + else: + # Handle single DataArray + for s in seasons: + dataarrays.append(pf.seasonal_mean(ds, season=s, is_climo=True)) + + # Combine seasonal means + ds_seasonal = xr.concat(dataarrays, dim='season') + ds_seasonal['season'] = seasons + ds_seasonal = ds_seasonal.transpose('lat', 'lon', 'season') + + return ds_seasonal \ No newline at end of file diff --git a/scripts/plotting/global_latlon_map.py b/scripts/plotting/global_latlon_map.py index c44fb20f9..c120a4281 100644 --- a/scripts/plotting/global_latlon_map.py +++ b/scripts/plotting/global_latlon_map.py @@ -11,27 +11,16 @@ plot_file_op Check on status of output plot file. """ -#Import standard modules: -import os +# Import standard modules: from pathlib import Path import numpy as np -import xarray as xr -import xesmf as xe -import warnings # use to warn user about missing files. - -# Import plotting modules: -import matplotlib as mpl -import matplotlib.pyplot as plt - -import cartopy.crs as ccrs -import cartopy.feature as cfeature -from cartopy.util import add_cyclic_point -from cartopy.mpl.ticker import LongitudeFormatter, LatitudeFormatter +import warnings + +# Import local modules: import plotting_functions as pf +from .aod_latlon import aod_latlon -# Warnings -import warnings # use to warn user about missing files. -# - Format warning messages: +# Format warning messages: def my_formatwarning(msg, *args, **kwargs): """Issue `msg` as warning.""" return str(msg) + '\n' @@ -92,251 +81,128 @@ def global_latlon_map(adfobj): Checks on pressure level dimension """ - #Notify user that script has started: msg = "\n Generating lat/lon maps..." print(f"{msg}\n {'-' * (len(msg)-3)}") - # - # Use ADF api to get all necessary information - # - var_list = adfobj.diag_var_list - #Special ADF variable which contains the output paths for - #all generated plots and tables for each case: - plot_locations = adfobj.plot_location - - #Grab case years - syear_cases = adfobj.climo_yrs["syears"] - eyear_cases = adfobj.climo_yrs["eyears"] - - #Grab baseline years (which may be empty strings if using Obs): - syear_baseline = adfobj.climo_yrs["syear_baseline"] - eyear_baseline = adfobj.climo_yrs["eyear_baseline"] + # Get configuration + config = get_plot_config(adfobj) + + # Process regular variables + for var in adfobj.diag_var_list: + process_variable(adfobj, var, **config) + + # Handle AOD special case + if "AODVISdn" in adfobj.diag_var_list: + print("\tRunning AOD panel diagnostics against MERRA and MODIS...") + aod_latlon(adfobj) + + print(" ...lat/lon maps have been generated successfully.") - res = adfobj.variable_defaults # will be dict of variable-specific plot preferences - # or an empty dictionary if use_defaults was not specified in YAML. - - #Set plot file type: - # -- this should be set in basic_info_dict, but is not required - # -- So check for it, and default to png - basic_info_dict = adfobj.read_config_var("diag_basic_info") - plot_type = basic_info_dict.get('plot_type', 'png') - print(f"\t NOTE: Plot type is set to {plot_type}") - - # check if existing plots need to be redone - redo_plot = adfobj.get_basic_info('redo_plot') - print(f"\t NOTE: redo_plot is set to {redo_plot}") - #----------------------------------------- - - #Determine if user wants to plot 3-D variables on - #pressure levels: - pres_levs = adfobj.get_basic_info("plot_press_levels") - - weight_season = True #always do seasonal weighting - - #Set seasonal ranges: - seasons = {"ANN": np.arange(1,13,1), - "DJF": [12, 1, 2], - "JJA": [6, 7, 8], - "MAM": [3, 4, 5], - "SON": [9, 10, 11] - } - - # probably want to do this one variable at a time: - for var in var_list: - #Notify user of variable being plotted: - print(f"\t - lat/lon maps for {var}") - - # Check res for any variable specific options that need to be used BEFORE going to the plot: - if var in res: - vres = res[var] - #If found then notify user, assuming debug log is enabled: - adfobj.debug_log(f"global_latlon_map: Found variable defaults for {var}") - - #Extract category (if available): - web_category = vres.get("category", None) - else: - vres = {} - web_category = None - #End if +def process_variable(adfobj, var, seasons, pres_levs, plot_type, redo_plot) + vres = adfobj.variable_defaults.get(var, {}) + web_category = vres.get("category", None) # For global maps, also set the central longitude: # can be specified in adfobj basic info as 'central_longitude' or supplied as a number, # otherwise defaults to 180 vres['central_longitude'] = pf.get_central_longitude(adfobj) - # load reference data (observational or baseline) - if not adfobj.compare_obs: - base_name = adfobj.data.ref_case_label - else: - if var not in adfobj.data.ref_var_nam: - dmsg = f"\t WARNING: No obs data found for variable `{var}`, global lat/lon mean plotting skipped." - adfobj.debug_log(dmsg) - print(dmsg) - continue - else: - base_name = adfobj.data.ref_labels[var] - - # Gather reference variable data - odata = adfobj.data.load_reference_regrid_da(base_name, var) - + # Load reference data + odata = load_reference_data(adfobj, var) if odata is None: - dmsg = f"\t WARNING: No regridded test file for {base_name} for variable `{var}`, global lat/lon mean plotting skipped." - adfobj.debug_log(dmsg) - continue - - o_has_dims = pf.validate_dims(odata, ["lat", "lon", "lev"]) # T iff dims are (lat,lon) -- can't plot unless we have both - if (not o_has_dims['has_lat']) or (not o_has_dims['has_lon']): - print(f"\t WARNING: skipping global map for {var} as REFERENCE does not have both lat and lon") continue #Loop over model cases: for case_idx, case_name in enumerate(adfobj.data.case_names): + process_case(adfobj, case_name, case_idx, var, odata, + seasons, pres_levs, plot_type, redo_plot, + vres, web_category) - #Set case nickname: - case_nickname = adfobj.data.test_nicknames[case_idx] - #Set output plot location: - plot_loc = Path(plot_locations[case_idx]) - #Check if plot output directory exists, and if not, then create it: - if not plot_loc.is_dir(): - print(f" {plot_loc} not found, making new directory") - plot_loc.mkdir(parents=True) +def load_reference_data(adfobj, var): + """Load and validate reference data.""" + if not adfobj.compare_obs: + base_name = adfobj.data.ref_case_label + else: + if var not in adfobj.data.ref_var_nam: + dmsg = f"\t WARNING: No obs data found for variable `{var}`, global lat/lon mean plotting skipped." + adfobj.debug_log(dmsg) + print(dmsg) return None + base_name = adfobj.data.ref_labels[var] + + odata = adfobj.data.load_reference_regrid_da(base_name, var) + if odata is None: + print(f"\t WARNING: No reference data found for {var}") + return None + + o_has_dims = pf.validate_dims(odata, ["lat", "lon", "lev"]) + if (not o_has_dims['has_lat']) or (not o_has_dims['has_lon']): + print(f"\t WARNING: Reference data missing lat/lon dimensions") + return None + + return odata - #Load re-gridded model files: - mdata = adfobj.data.load_regrid_da(case_name, var) - #Skip this variable/case if the regridded climo file doesn't exist: - if mdata is None: - dmsg = f"\t WARNING: No regridded test file for {case_name} for variable `{var}`, global lat/lon mean plotting skipped." - adfobj.debug_log(dmsg) - continue +def process_case(adfobj, case_name, case_idx, var, odata, seasons, + pres_levs, plot_type, redo_plot, vres, web_category): + """Process individual case data and generate plots.""" + plot_loc = Path(adfobj.plot_location[case_idx]) + plot_loc.mkdir(parents=True, exist_ok=True) - #Determine dimensions of variable: - has_dims = pf.validate_dims(mdata, ["lat", "lon", "lev"]) - if (not has_dims['has_lat']) or (not has_dims['has_lon']): - print(f"\t WARNING: skipping global map for {var} for case {case_name} as it does not have both lat and lon") - continue - else: # i.e., has lat&lon - if (has_dims['has_lev']) and (not pres_levs): - print(f"\t WARNING: skipping global map for {var} as it has more than lev dimension, but no pressure levels were provided") - continue - - # Check output file. If file does not exist, proceed. - # If file exists: - # if redo_plot is true: delete it now and make plot - # if redo_plot is false: add to website and move on - doplot = {} - - if not has_dims['has_lev']: - for s in seasons: - plot_name = plot_loc / f"{var}_{s}_LatLon_Mean.{plot_type}" - doplot[plot_name] = plot_file_op(adfobj, plot_name, var, case_name, s, web_category, redo_plot, "LatLon") - else: - for pres in pres_levs: - for s in seasons: - plot_name = plot_loc / f"{var}_{pres}hpa_{s}_LatLon_Mean.{plot_type}" - doplot[plot_name] = plot_file_op(adfobj, plot_name, f"{var}_{pres}hpa", case_name, s, web_category, redo_plot, "LatLon") - if not any(value is None for value in doplot.values()): - print(f"\t INFO: All plots exist for {var}. Redo is {redo_plot}. Existing plots added to website data. Continue.") - continue + mdata = adfobj.data.load_regrid_da(case_name, var) + if mdata is None: + return - #Create new dictionaries: - mseasons = {} - oseasons = {} - dseasons = {} # hold the differences - pseasons = {} # hold percent change - - if not has_dims['has_lev']: # strictly 2-d data - - #Loop over season dictionary: - for s in seasons: - plot_name = plot_loc / f"{var}_{s}_LatLon_Mean.{plot_type}" - if doplot[plot_name] is None: - continue - - if weight_season: - mseasons[s] = pf.seasonal_mean(mdata, season=s, is_climo=True) - oseasons[s] = pf.seasonal_mean(odata, season=s, is_climo=True) - else: - #Just average months as-is: - mseasons[s] = mdata.sel(time=seasons[s]).mean(dim='time') - oseasons[s] = odata.sel(time=seasons[s]).mean(dim='time') - #End if - - # difference: each entry should be (lat, lon) - dseasons[s] = mseasons[s] - oseasons[s] - - # percent change - pseasons[s] = (mseasons[s] - oseasons[s]) / np.abs(oseasons[s]) * 100.0 #relative change - - pf.plot_map_and_save(plot_name, case_nickname, adfobj.data.ref_nickname, - [syear_cases[case_idx],eyear_cases[case_idx]], - [syear_baseline,eyear_baseline], - mseasons[s], oseasons[s], dseasons[s], pseasons[s], - obs=adfobj.compare_obs, **vres) - - #Add plot to website (if enabled): - adfobj.add_website_data(plot_name, var, case_name, category=web_category, - season=s, plot_type="LatLon") - - else: # => pres_levs has values, & we already checked that lev is in mdata (has_lev) - - for pres in pres_levs: - - #Check that the user-requested pressure level - #exists in the model data, which should already - #have been interpolated to the standard reference - #pressure levels: - if (not (pres in mdata['lev'])) or (not (pres in odata['lev'])): - print(f"\t WARNING: plot_press_levels value '{pres}' not present in {var} [test: {(pres in mdata['lev'])}, ref: {pres in odata['lev']}], so skipping.") - continue - - #Loop over seasons: - for s in seasons: - plot_name = plot_loc / f"{var}_{pres}hpa_{s}_LatLon_Mean.{plot_type}" - if doplot[plot_name] is None: - continue - - if weight_season: - mseasons[s] = pf.seasonal_mean(mdata, season=s, is_climo=True) - oseasons[s] = pf.seasonal_mean(odata, season=s, is_climo=True) - else: - #Just average months as-is: - mseasons[s] = mdata.sel(time=seasons[s]).mean(dim='time') - oseasons[s] = odata.sel(time=seasons[s]).mean(dim='time') - #End if - - # difference: each entry should be (lat, lon) - dseasons[s] = mseasons[s] - oseasons[s] - - # percent change - pseasons[s] = (mseasons[s] - oseasons[s]) / np.abs(oseasons[s]) * 100.0 #relative change - - pf.plot_map_and_save(plot_name, case_nickname, adfobj.data.ref_nickname, - [syear_cases[case_idx],eyear_cases[case_idx]], - [syear_baseline,eyear_baseline], - mseasons[s].sel(lev=pres), oseasons[s].sel(lev=pres), dseasons[s].sel(lev=pres), - pseasons[s].sel(lev=pres), - obs=adfobj.compare_obs, **vres) - - #Add plot to website (if enabled): - adfobj.add_website_data(plot_name, f"{var}_{pres}hpa", case_name, category=web_category, - season=s, plot_type="LatLon") - #End for (seasons) - #End for (pressure levels) - #End if (plotting pressure levels) - #End for (case loop) - #End for (variable loop) - - # Check for AOD, and run the 4-panel diagnostics against MERRA and MODIS - if "AODVISdn" in var_list: - print("\tRunning AOD panel diagnostics against MERRA and MODIS...") - aod_latlon(adfobj) + has_dims = pf.validate_dims(mdata, ["lat", "lon", "lev"]) + if not pf.lat_lon_validate_dims(mdata): + print(f"\t WARNING: Model data missing lat/lon dimensions") + return - #Notify user that script has ended: - print(" ...lat/lon maps have been generated successfully.") + # Check pressure levels if 3D data + if has_dims['has_lev'] and not pres_levs: + print(f"\t WARNING: 3D variable found but no pressure levels specified") + return + + process_plots(adfobj, mdata, odata, case_name, case_idx, + var, seasons, pres_levs, plot_loc, plot_type, + redo_plot, vres, web_category, has_dims) + + +def get_plot_config(adfobj): + """Get plotting configuration from ADF object.""" + return { + 'seasons': { + "ANN": np.arange(1,13,1), + "DJF": [12, 1, 2], + "JJA": [6, 7, 8], + "MAM": [3, 4, 5], + "SON": [9, 10, 11] + }, + 'plot_type': adfobj.read_config_var("diag_basic_info").get('plot_type', 'png'), + 'redo_plot': adfobj.get_basic_info('redo_plot'), + 'pres_levs': adfobj.get_basic_info("plot_press_levels") + } + + +def process_seasonal_data(mdata, odata, season, weight_season=True): + """Helper function to calculate seasonal means and differences.""" + if weight_season: + mseason = pf.seasonal_mean(mdata, season=season, is_climo=True) + oseason = pf.seasonal_mean(odata, season=season, is_climo=True) + else: + mseason = mdata.sel(time=seasons[s]).mean(dim='time') + oseason = odata.sel(time=seasons[s]).mean(dim='time') + + # Calculate differences + dseason = mseason - oseason + + # Calculate percent change + pseason = (mseason - oseason) / np.abs(oseason) * 100.0 + pseason = pseason.where(np.isfinite(pseason), np.nan) + + return mseason, oseason, dseason, pseason def plot_file_op(adfobj, plot_name, var, case_name, season, web_category, redo_plot, plot_type): @@ -392,533 +258,168 @@ def plot_file_op(adfobj, plot_name, var, case_name, season, web_category, redo_p return False # False tells caller that file exists and not to overwrite else: return True -######## - - -def aod_latlon(adfobj): - """ - Function to gather data and plot parameters to plot a panel plot of model vs observation - difference and percent difference. - - Calculate the seasonal means for DJF, MAM, JJA, SON for model and obs datasets - NOTE: The model lat/lons must be on the same grid as the observations. If they are not, they will be - regridded to match both the MERRA and MODIS observation dataset using helper function 'regrid_to_obs' - For details about spatial coordiantes of obs datasets, see /glade/campaign/cgd/amp/amwg/ADF_obs/: - - MERRA2_192x288_AOD_2001-2020_climo.nc - - MOD08_M3_192x288_AOD_2001-2020_climo.nc +def process_plots(adfobj, mdata, odata, case_name, case_idx, var, seasons, + pres_levs, plot_loc, plot_type, redo_plot, vres, web_category, has_dims): + """Process and generate plots for different seasons and pressure levels. + + Parameters + ---------- + adfobj : AdfDiag + The diagnostics object containing configuration + mdata : xarray.DataArray + Model data + odata : xarray.DataArray + Reference/observation data + case_name : str + Name of current case + case_idx : int + Index of current case + var : str + Variable name + seasons : dict + Dictionary of season definitions + pres_levs : list + Pressure levels to plot + plot_loc : Path + Output plot directory + plot_type : str + Plot file type (e.g. 'png') + redo_plot : bool + Whether to regenerate existing plots + vres : dict + Variable-specific plot settings + web_category : str + Category for website organization + has_dims : dict + Dictionary indicating which dimensions exist in data + + Returns + ------- + None """ - - var = "AODVISdn" - season_abbr = ['Dec-Jan-Feb', 'Mar-Apr-May', 'Jun-Jul-Aug', 'Sep-Oct-Nov'] - # Define a list of season labels - seasons = ['DJF', 'MAM', 'JJA', 'SON'] - - test_case_names = adfobj.get_cam_info('cam_case_name', required=True) - # load reference data (observational or baseline) - if not adfobj.compare_obs: - base_name = adfobj.data.ref_case_label - case_names = test_case_names + [base_name] - else: - case_names = test_case_names - - #Grab all case nickname(s) - test_nicknames = adfobj.case_nicknames["test_nicknames"] - base_nickname = adfobj.case_nicknames["base_nickname"] - case_nicknames = test_nicknames + [base_nickname] - - res = adfobj.variable_defaults # will be dict of variable-specific plot preferences - # or an empty dictionary if use_defaults was not specified in YAML. - res_aod_diags = res["aod_diags"] - plot_params = res_aod_diags["plot_params"] - plot_params_relerr = res_aod_diags["plot_params_relerr"] - - # Observational Datasets - #----------------------- - # Round lat/lons to 5 decimal places - # NOTE: this is neccessary due to small fluctuations in insignificant decimal places - # in lats/lons between models and these obs data sets. The model cases will also - # be rounded in turn. - obs_dir = adfobj.get_basic_info("obs_data_loc") - file_merra2 = os.path.join(obs_dir, 'MERRA2_192x288_AOD_2001-2020_climo.nc') - file_mod08_m3 = os.path.join(obs_dir, 'MOD08_M3_192x288_AOD_2001-2020_climo.nc') - - if (not Path(file_merra2).is_file()) or (not Path(file_mod08_m3).is_file()): - print("\t WARNING: AOD Panel plots not made, missing MERRA2 and/or MODIS file") + # Get case nickname and years + case_nickname = adfobj.data.test_nicknames[case_idx] + syear_cases = adfobj.climo_yrs["syears"] + eyear_cases = adfobj.climo_yrs["eyears"] + syear_baseline = adfobj.climo_yrs["syear_baseline"] + eyear_baseline = adfobj.climo_yrs["eyear_baseline"] + + # Check if files exist and build doplot dict + doplot = check_existing_plots(adfobj, var, plot_loc, plot_type, + case_name, seasons, pres_levs, + has_dims, web_category, redo_plot) + + if not any(value is None for value in doplot.values()): + print(f"\t INFO: All plots exist for {var}. Redo is {redo_plot}. Existing plots added to website data.") return - ds_merra2 = xr.open_dataset(file_merra2) - ds_merra2 = ds_merra2['TOTEXTTAU'] - ds_merra2['lon'] = ds_merra2['lon'].round(5) - ds_merra2['lat'] = ds_merra2['lat'].round(5) - - ds_mod08_m3 = xr.open_dataset(file_mod08_m3) - ds_mod08_m3 = ds_mod08_m3['AOD_550_Dark_Target_Deep_Blue_Combined_Mean_Mean'] - ds_mod08_m3['lon'] = ds_mod08_m3['lon'].round(5) - ds_mod08_m3['lat'] = ds_mod08_m3['lat'].round(5) - - ds_merra2_season = monthly_to_seasonal(ds_merra2) - ds_merra2_season['lon'] = ds_merra2_season['lon'].round(5) - ds_merra2_season['lat'] = ds_merra2_season['lat'].round(5) - - ds_mod08_m3_season = monthly_to_seasonal(ds_mod08_m3) - ds_mod08_m3_season['lon'] = ds_mod08_m3_season['lon'].round(5) - ds_mod08_m3_season['lat'] = ds_mod08_m3_season['lat'].round(5) - - ds_obs = [ds_mod08_m3_season, ds_merra2_season] - obs_lat_shape = ds_obs[0]['lat'].shape[0] - obs_lon_shape = ds_obs[0]['lon'].shape[0] - obs_titles = ["TERRA MODIS", "MERRA2"] - - # Model Case Datasets - #----------------------- - ds_cases = [] - - for case in test_case_names: - #Load re-gridded model files: - ds_case = adfobj.data.load_climo_da(case, var) - - #Skip this variable/case if the climo file doesn't exist: - if ds_case is None: - dmsg = f"\t WARNING: No test climo file for {case} for variable `{var}`, global lat/lon plots skipped." - adfobj.debug_log(dmsg) + # Initialize seasonal data dictionaries + mseasons = {} + oseasons = {} + dseasons = {} + pseasons = {} + + if not has_dims['has_lev']: + # Process 2D data + process_2d_plots(adfobj, mdata, odata, case_name, case_nickname, + var, seasons, plot_loc, plot_type, doplot, + mseasons, oseasons, dseasons, pseasons, + syear_cases[case_idx], eyear_cases[case_idx], + syear_baseline, eyear_baseline, + web_category, vres) + else: + # Process 3D data with pressure levels + process_3d_plots(adfobj, mdata, odata, case_name, case_nickname, + var, seasons, pres_levs, plot_loc, plot_type, doplot, + mseasons, oseasons, dseasons, pseasons, + syear_cases[case_idx], eyear_cases[case_idx], + syear_baseline, eyear_baseline, + web_category, vres) + +def check_existing_plots(adfobj, var, plot_loc, plot_type, case_name, + seasons, pres_levs, has_dims, web_category, redo_plot): + """Check which plots need to be generated.""" + doplot = {} + + if not has_dims['has_lev']: + for s in seasons: + plot_name = plot_loc / f"{var}_{s}_LatLon_Mean.{plot_type}" + doplot[plot_name] = plot_file_op(adfobj, plot_name, var, + case_name, s, web_category, + redo_plot, "LatLon") + else: + for pres in pres_levs: + for s in seasons: + plot_name = plot_loc / f"{var}_{pres}hpa_{s}_LatLon_Mean.{plot_type}" + doplot[plot_name] = plot_file_op(adfobj, plot_name, + f"{var}_{pres}hpa", + case_name, s, web_category, + redo_plot, "LatLon") + return doplot + +def process_2d_plots(adfobj, mdata, odata, case_name, case_nickname, + var, seasons, plot_loc, plot_type, doplot, + mseasons, oseasons, dseasons, pseasons, + syear_case, eyear_case, syear_baseline, eyear_baseline, + web_category, vres): + """Process and generate 2D plots.""" + for s in seasons: + plot_name = plot_loc / f"{var}_{s}_LatLon_Mean.{plot_type}" + if doplot[plot_name] is None: continue - else: - # Round lat/lons so they match obs - # NOTE: this is neccessary due to small fluctuations in insignificant decimal places - # that raise an error due to non-exact difference calculations. - # Rounding all datasets to 5 places ensures the proper difference calculation - ds_case['lon'] = ds_case['lon'].round(5) - ds_case['lat'] = ds_case['lat'].round(5) - case_lat_shape = ds_case['lat'].shape[0] - case_lon_shape = ds_case['lon'].shape[0] - - # Check if the lats/lons are same as the first supplied observation set - if case_lat_shape == obs_lat_shape: - case_lat = True - else: - err_msg = "AOD 4-panel plot:\n" - err_msg += f"\t WARNING: The lat values don't match between obs and '{case}'\n" - err_msg += f"\t - {case} lat shape: {case_lat_shape} and " - err_msg += f"obs lat shape: {obs_lat_shape}" - adfobj.debug_log(err_msg) - print(err_msg) - case_lat = False - # End if - - if case_lon_shape == obs_lon_shape: - case_lon = True - else: - err_msg = "AOD 4-panel plot:\n" - err_msg += f"\t WARNING: The lon values don't match between obs and '{case}'\n" - err_msg += f"\t - {case} lon shape: {case_lon_shape} and " - err_msg += f"obs lon shape: {obs_lon_shape}" - adfobj.debug_log(err_msg) - print(err_msg) - case_lon = False - # End if - # Check to make sure spatial dimensions are compatible - if (case_lat) and (case_lon): - # Calculate seasonal means - ds_case_season = monthly_to_seasonal(ds_case) - ds_case_season['lon'] = ds_case_season['lon'].round(5) - ds_case_season['lat'] = ds_case_season['lat'].round(5) - ds_cases.append(ds_case_season) - else: - # Regrid the model data to obs - #NOTE: first argument is the model to be regridded, second is the obs - # to be regridded to - ds_case_regrid = regrid_to_obs(adfobj, ds_case, ds_obs[0]) - - ds_case_season = monthly_to_seasonal(ds_case_regrid) - ds_case_season['lon'] = ds_case_season['lon'].round(5) - ds_case_season['lat'] = ds_case_season['lat'].round(5) - ds_cases.append(ds_case_season) - # End if - # End if - - # load reference data (observational or baseline) - if not adfobj.compare_obs: + # Calculate seasonal means and differences + mseasons[s], oseasons[s], dseasons[s], pseasons[s] = \ + process_seasonal_data(mdata, odata, s) + + # Generate plot + pf.plot_map_and_save(plot_name, case_nickname, adfobj.data.ref_nickname, + [syear_case, eyear_case], + [syear_baseline, eyear_baseline], + mseasons[s], oseasons[s], dseasons[s], pseasons[s], + obs=adfobj.compare_obs, **vres) + + # Add to website + adfobj.add_website_data(plot_name, var, case_name, + category=web_category, + season=s, plot_type="LatLon") + +def process_3d_plots(adfobj, mdata, odata, case_name, case_nickname, + var, seasons, pres_levs, plot_loc, plot_type, doplot, + mseasons, oseasons, dseasons, pseasons, + syear_case, eyear_case, syear_baseline, eyear_baseline, + web_category, vres): + """Process and generate 3D plots with pressure levels.""" + for pres in pres_levs: + # Validate pressure level exists + if (not (pres in mdata['lev'])) or (not (pres in odata['lev'])): + print(f"\t WARNING: plot_press_levels value '{pres}' not present " + f"in {var} [test: {(pres in mdata['lev'])}, " + f"ref: {pres in odata['lev']}], so skipping.") + continue - # Get baseline case name - base_name = adfobj.data.ref_case_label - - # Gather reference variable data - ds_base = adfobj.data.load_reference_climo_da(base_name, var) - if ds_base is None: - dmsg = f"\t WARNING: No baseline climo file for {base_name} for variable `{var}`, global lat/lon plots skipped." - adfobj.debug_log(dmsg) - else: - # Round lat/lons so they match obs - # NOTE: this is neccessary due to small fluctuations in insignificant decimal places - # that raise an error due to non-exact difference calculations. - # Rounding all datasets to 5 places ensures the proper difference calculation - ds_base['lon'] = ds_base['lon'].round(5) - ds_base['lat'] = ds_base['lat'].round(5) - base_lat_shape = ds_base['lat'].shape[0] - base_lon_shape = ds_base['lon'].shape[0] - - # Check if the lats/lons are same as the first supplied observation set - if base_lat_shape == obs_lat_shape: - base_lat = True - else: - err_msg = "AOD 4-panel plot:\n" - err_msg += f"\t WARNING: The lat values don't match between obs and '{base_name}'\n" - err_msg += f"\t - {base_name} lat shape: {base_lat_shape} and " - err_msg += f"obs lat shape: {obs_lat_shape}" - adfobj.debug_log(err_msg) - print(err_msg) - base_lat = False - # End if - - if base_lon_shape == obs_lon_shape: - base_lon = True - else: - err_msg = "AOD 4-panel plot:\n" - err_msg += f"\t WARNING: The lon values don't match between obs and '{base_name}'\n" - err_msg += f"\t - {base_name} lon shape: {base_lon_shape} and " - err_msg += f"obs lon shape: {obs_lon_shape}" - adfobj.debug_log(err_msg) - print(err_msg) - base_lon = False - # End if - - # Check to make sure spatial dimensions are compatible - if (base_lat) and (base_lon): - # Calculate seasonal means - ds_base_season = monthly_to_seasonal(ds_base) - ds_base_season['lon'] = ds_base_season['lon'].round(5) - ds_base_season['lat'] = ds_base_season['lat'].round(5) - ds_cases.append(ds_base_season) - else: - # Regrid the model data to obs - #NOTE: first argument is the model to be regridded, second is the obs - # to be regridded to - ds_base_regrid = regrid_to_obs(adfobj, ds_base, ds_obs[0]) - - ds_base_season = monthly_to_seasonal(ds_base_regrid) - ds_base_season['lon'] = ds_base_season['lon'].round(5) - ds_base_season['lat'] = ds_base_season['lat'].round(5) - ds_cases.append(ds_base_season) - # End if - # End if - # Number of relevant cases - case_num = len(ds_cases) - - # 4-Panel global lat/lon plots - #----------------------------- - # NOTE: This loops over all obs and available cases, so just - # make lists to keepo track of details for each case vs obs matchup - # Plots: - # - Difference of seasonal avg of case minus seasonal avg of observation - # - Percent Difference of seasonal avg of case minus seasonal avg of observation - - # Loop over each observation dataset first - for i_obs,ds_ob in enumerate(ds_obs): - for i_s,season in enumerate(seasons): - # Plot title list - plot_titles = [] - # Calculated data list - data = [] - # Plot parameter list - params = [] - # Plot type list, ie difference or percent difference - types = [] - # Model case name list - case_name_list = [] - - # Get observation short name - obs_name = obs_titles[i_obs] - - # Get seasonal abbriviation - chem_season = season_abbr[i_s] - - # Then loop over each available model case - for i_case,ds_case in enumerate(ds_cases): - case_nickname = case_nicknames[i_case] - - # Difference with obs - case_field = ds_case.sel(season=season) - ds_ob.sel(season=season) - plot_titles.append(f'{case_nickname} - {obs_name}\nAOD 550 nm - ' + chem_season) - data.append(case_field) - params.append(plot_params) - types.append("Diff") - case_name_list.append(case_names[i_case]) - - # Percent difference with obs - field_relerr = 100 * case_field / ds_ob.sel(season=season) - field_relerr = np.clip(field_relerr, -100, 100) - plot_titles.append(f'Percent Diff {case_nickname} - {obs_name}\nAOD 550 nm - ' + chem_season) - data.append(field_relerr) - params.append(plot_params_relerr) - types.append("Percent Diff") - case_name_list.append(case_names[i_case]) - # End for - - # Create 4-panel plot for season - aod_panel_latlon(adfobj, plot_titles, params, data, season, obs_name, case_name_list, case_num, types, symmetric=True) - # End for - # End for - - -######################################## -# Helper functions for AOD 4-panel plots -######################################## - -def monthly_to_seasonal(ds,obs=False): - ds_season = xr.Dataset( - coords={'lat': ds.coords['lat'], 'lon': ds.coords['lon'], - 'season': np.arange(4)}) - da_season = xr.DataArray( - coords=ds_season.coords, dims=['lat', 'lon', 'season']) - - # Create a list of DataArrays - dataarrays = [] - # Define a list of season labels - seasons = ['DJF', 'MAM', 'JJA', 'SON'] - - if obs: - for varname in ds: - if '_n' not in varname: - ds_season = xr.zeros_like(da_season) - for s in seasons: - dataarrays.append(pf.seasonal_mean(ds, season=s, is_climo=True)) - else: for s in seasons: - dataarrays.append(pf.seasonal_mean(ds, season=s, is_climo=True)) - - # Use xr.concat to combine along a new 'season' dimension - ds_season = xr.concat(dataarrays, dim='season') - - # Assign the 'season' labels to the new 'season' dimension - ds_season['season'] = seasons - ds_season = ds_season.transpose('lat', 'lon', 'season') - - return ds_season -####### - - -def aod_panel_latlon(adfobj, plot_titles, plot_params, data, season, obs_name, case_name, case_num, types, symmetric=False): - """ - Function to plot a panel plot of model vs observation difference and percent difference - - This will be a 4-panel plot if model vs model run: - - Top left is test model minus obs - - Top right is baseline model minus obs - - Bottom left is test model minus obs percent difference - - Bottom right is baseline model minus obs percent difference - - This will be a 2-panel plot if model vs obs run: - - Top is test model minus obs - - Bottom is test model minus obs percent difference - - NOTE: Individual plots of the panel plots will be created and saved to plotting location(s) - but will not be published to the webpage (if enabled) - """ - #Set plot details: - # -- this should be set in basic_info_dict, but is not required - # -- So check for it, and default to png - basic_info_dict = adfobj.read_config_var("diag_basic_info") - file_type = basic_info_dict.get('plot_type', 'png') - plot_dir = adfobj.plot_location[0] - - # check if existing plots need to be redone - redo_plot = adfobj.get_basic_info('redo_plot') - - # Save the panel figure - plot_name = f'AOD_diff_{obs_name.replace(" ","_")}_{season}_LatLon_Mean.{file_type}' - plotfile = Path(plot_dir) / plot_name - - # Check redo_plot. If set to True: remove old plot, if it already exists: - if (not redo_plot) and plotfile.is_file(): - adfobj.debug_log(f"'{plotfile}' exists and clobber is false.") - #Add already-existing plot to website (if enabled): - adfobj.add_website_data(plotfile, f'AOD_diff_{obs_name.replace(" ","_")}', None, - season=season, multi_case=True, plot_type="LatLon", category="4-Panel AOD Diags") + plot_name = plot_loc / f"{var}_{pres}hpa_{s}_LatLon_Mean.{plot_type}" + if doplot[plot_name] is None: + continue - # Exit - return - else: - if plotfile.is_file(): - plotfile.unlink() - # End if - # End if - - # create figure: - fig = plt.figure(figsize=(7*case_num,10)) - proj = ccrs.PlateCarree() - - # LAYOUT WITH GRIDSPEC - plot_len = int(3*case_num) - gs = mpl.gridspec.GridSpec(2*case_num, plot_len, wspace=0.5, hspace=0.0) - gs.tight_layout(fig) - - axs = [] - for i in range(case_num): - start = i * 3 - end = (i + 1) * 3 - axs.append(plt.subplot(gs[0:case_num, start:end], projection=proj)) - axs.append(plt.subplot(gs[case_num:, start:end], projection=proj)) - # End for - - # formatting for tick labels - lon_formatter = LongitudeFormatter(number_format='0.0f', - degree_symbol='', - dateline_direction_label=False) - lat_formatter = LatitudeFormatter(number_format='0.0f', - degree_symbol='') - - # Loop over each data set - for i,field in enumerate(data): - # Set up sub plots for main panel plot - ind_fig, ind_ax = plt.subplots(1, 1, figsize=((7*case_num)/2,10/2),subplot_kw={'projection': proj}) - - lon_values = field.lon.values - lat_values = field.lat.values - - # Get field plot paramters - plot_param = plot_params[i] - - # Define plot levels - levels = np.linspace( - plot_param['range_min'], plot_param['range_max'], - plot_param['nlevel'], endpoint=True) - if 'augment_levels' in plot_param: - levels = sorted(np.append( - levels, np.array(plot_param['augment_levels']))) - # End if - - if field.ndim > 2: - print(f"Required 2d lat/lon coordinates, got {field.ndim}d") - emg = "AOD panel plot:\n" - emg += f"\t WARNING: Too many dimensions for {case_name}. Needs 2 (lat/lon) but got {field.ndim}" - adfobj.debug_log(emg) - print(f"{emg} ") - return - # End if - - # Get data - field_values = field.values[:,:] - field_values, lon_values = add_cyclic_point(field_values, coord=lon_values) - lon_mesh, lat_mesh = np.meshgrid(lon_values, lat_values) - field_mean = np.nanmean(field_values) - - # Set plot details - extend_option = 'both' if symmetric else 'max' - - if 'colormap' in plot_param: - cmap_option = plot_param['colormap'] if symmetric else plt.cm.turbo - else: - cmap_option = plt.cm.bwr if symmetric else plt.cm.turbo - - img = axs[i].contourf(lon_mesh, lat_mesh, field_values, - levels, cmap=cmap_option, extend=extend_option, - transform_first=True, - transform=ccrs.PlateCarree()) - ind_img = ind_ax.contourf(lon_mesh, lat_mesh, field_values, - levels, cmap=cmap_option, extend=extend_option, - transform_first=True, - transform=ccrs.PlateCarree()) - - axs[i].set_facecolor('gray') - ind_ax.set_facecolor('gray') - axs[i].coastlines() - ind_ax.coastlines() - - # Set plot titles - axs[i].set_title(plot_titles[i] + (' Mean %.2g' % field_mean),fontsize=10) - ind_ax.set_title(plot_titles[i] + (' Mean %.2g' % field_mean),fontsize=10) - - # Colorbar options - cbar = plt.colorbar(img, orientation='horizontal', pad=0.05) - ind_cbar = plt.colorbar(ind_img, orientation='horizontal', pad=0.05) - - if 'ticks' in plot_param: - cbar.set_ticks(plot_param['ticks']) - ind_cbar.set_ticks(plot_param['ticks']) - if 'tick_labels' in plot_param: - cbar.ax.set_xticklabels(plot_param['tick_labels']) - ind_cbar.ax.set_xticklabels(plot_param['tick_labels']) - cbar.ax.tick_params(labelsize=6) - - # Save the individual figure - pbase = f'AOD_{case_name[i]}_vs_{obs_name.replace(" ","_")}_{types[i].replace(" ","_")}' - ind_plotfile = f'{pbase}_{season}_LatLon_Mean.{file_type}' - ind_png_file = Path(plot_dir) / ind_plotfile - ind_fig.savefig(f'{ind_png_file}', bbox_inches='tight', dpi=300) - plt.close(ind_fig) - # End for - - # Save the panel figure - plot_name = f'AOD_diff_{obs_name.replace(" ","_")}_{season}_LatLon_Mean.{file_type}' - plotfile = Path(plot_dir) / plot_name - - # Save figure and add to website if applicable - fig.savefig(plotfile, bbox_inches='tight', dpi=300) - adfobj.add_website_data(plotfile, f'AOD_diff_{obs_name.replace(" ","_")}', None, - season=season, multi_case=True, plot_type="LatLon", category="4-Panel AOD Diags") - - # Close the figure - plt.close(fig) -###### - - -def regrid_to_obs(adfobj, model_arr, obs_arr): - """ - Check if the model grid needs to be interpolated to the obs grid. If so, - use xesmf to regrid and return new dataset - """ - test_lons = model_arr.lon - test_lats = model_arr.lat - - obs_lons = obs_arr.lon - obs_lats = obs_arr.lat - - # Just set defaults for now - same_lats = True - same_lons = True - model_regrid_arr = None - - if obs_lons.shape == test_lons.shape: - try: - xr.testing.assert_equal(test_lons, obs_lons) - except AssertionError as e: - same_lons = False - err_msg = "AOD 4-panel plot:\n" - err_msg += "\t The lons ARE NOT the same" - adfobj.debug_log(err_msg) - try: - xr.testing.assert_equal(test_lats, obs_lats) - except AssertionError as e: - same_lats = False - err_msg = "AOD 4-panel plot:\n" - err_msg += "\t The lats ARE NOT the same" - adfobj.debug_log(err_msg) - else: - same_lats = False - same_lons = False - print("\tThe model lat/lon grid does not match the " \ - "obs grid.\n\t - Regridding to observation lats and lons") - - # QUESTION: will there ever be a scenario where we need to regrid only lats or lons?? - if (not same_lons) and (not same_lats): - # Make dummy array to be populated - ds_out = xr.Dataset( - { - "lat": (["lat"], obs_lats.values, {"units": "degrees_north"}), - "lon": (["lon"], obs_lons.values, {"units": "degrees_east"}), - } - ) - - # Regrid to the obs grid to make altered model grid - regridder = xe.Regridder(model_arr, ds_out, "bilinear", periodic=True) - model_regrid_arr = regridder(model_arr, keep_attrs=True) - - # Return the new interpolated model array - return model_regrid_arr -####### - -############## -#END OF SCRIPT \ No newline at end of file + # Calculate seasonal means and differences + mseasons[s], oseasons[s], dseasons[s], pseasons[s] = \ + process_seasonal_data(mdata, odata, s) + + # Generate plot + pf.plot_map_and_save(plot_name, case_nickname, adfobj.data.ref_nickname, + [syear_case, eyear_case], + [syear_baseline, eyear_baseline], + mseasons[s].sel(lev=pres), + oseasons[s].sel(lev=pres), + dseasons[s].sel(lev=pres), + pseasons[s].sel(lev=pres), + obs=adfobj.compare_obs, **vres) + + # Add to website + adfobj.add_website_data(plot_name, f"{var}_{pres}hpa", + case_name, category=web_category, + season=s, plot_type="LatLon") \ No newline at end of file From 88f5ffbf2debd06845984a2cc4d2272f53a5cab3 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 14 Mar 2025 14:43:46 -0600 Subject: [PATCH 16/91] fixed missing colon --- scripts/plotting/global_latlon_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plotting/global_latlon_map.py b/scripts/plotting/global_latlon_map.py index c120a4281..cfc15fcf7 100644 --- a/scripts/plotting/global_latlon_map.py +++ b/scripts/plotting/global_latlon_map.py @@ -99,7 +99,7 @@ def global_latlon_map(adfobj): print(" ...lat/lon maps have been generated successfully.") -def process_variable(adfobj, var, seasons, pres_levs, plot_type, redo_plot) +def process_variable(adfobj, var, seasons, pres_levs, plot_type, redo_plot): vres = adfobj.variable_defaults.get(var, {}) web_category = vres.get("category", None) From 13b1b266cbd267216a5f97427150ad3fbbe44bba Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 14 Mar 2025 14:44:58 -0600 Subject: [PATCH 17/91] fixed missing newline --- scripts/plotting/global_latlon_map.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/plotting/global_latlon_map.py b/scripts/plotting/global_latlon_map.py index cfc15fcf7..c80c8ca59 100644 --- a/scripts/plotting/global_latlon_map.py +++ b/scripts/plotting/global_latlon_map.py @@ -129,7 +129,8 @@ def load_reference_data(adfobj, var): if var not in adfobj.data.ref_var_nam: dmsg = f"\t WARNING: No obs data found for variable `{var}`, global lat/lon mean plotting skipped." adfobj.debug_log(dmsg) - print(dmsg) return None + print(dmsg) + return None base_name = adfobj.data.ref_labels[var] odata = adfobj.data.load_reference_regrid_da(base_name, var) From a2813566ef3530a7b291cce695700f6b1926baae Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 14 Mar 2025 14:48:04 -0600 Subject: [PATCH 18/91] fixed continue not in loop. return instead. --- scripts/plotting/global_latlon_map.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/plotting/global_latlon_map.py b/scripts/plotting/global_latlon_map.py index c80c8ca59..9bc512bdf 100644 --- a/scripts/plotting/global_latlon_map.py +++ b/scripts/plotting/global_latlon_map.py @@ -111,7 +111,8 @@ def process_variable(adfobj, var, seasons, pres_levs, plot_type, redo_plot): # Load reference data odata = load_reference_data(adfobj, var) if odata is None: - continue + print(f"[global_latlon_map][process_variable] finds no reference data.") + return #Loop over model cases: for case_idx, case_name in enumerate(adfobj.data.case_names): From c8ea1cfca2d2fc0ed357b7ca642f77160b16be23 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 14 Mar 2025 14:51:51 -0600 Subject: [PATCH 19/91] fixed several more typos --- scripts/plotting/global_latlon_map.py | 41 ++++++++++++++------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/scripts/plotting/global_latlon_map.py b/scripts/plotting/global_latlon_map.py index 9bc512bdf..a2ade38c5 100644 --- a/scripts/plotting/global_latlon_map.py +++ b/scripts/plotting/global_latlon_map.py @@ -100,25 +100,25 @@ def global_latlon_map(adfobj): def process_variable(adfobj, var, seasons, pres_levs, plot_type, redo_plot): - vres = adfobj.variable_defaults.get(var, {}) - web_category = vres.get("category", None) + vres = adfobj.variable_defaults.get(var, {}) + web_category = vres.get("category", None) - # For global maps, also set the central longitude: - # can be specified in adfobj basic info as 'central_longitude' or supplied as a number, - # otherwise defaults to 180 - vres['central_longitude'] = pf.get_central_longitude(adfobj) + # For global maps, also set the central longitude: + # can be specified in adfobj basic info as 'central_longitude' or supplied as a number, + # otherwise defaults to 180 + vres['central_longitude'] = pf.get_central_longitude(adfobj) - # Load reference data - odata = load_reference_data(adfobj, var) - if odata is None: - print(f"[global_latlon_map][process_variable] finds no reference data.") - return + # Load reference data + odata = load_reference_data(adfobj, var) + if odata is None: + print(f"[global_latlon_map][process_variable] finds no reference data.") + return - #Loop over model cases: - for case_idx, case_name in enumerate(adfobj.data.case_names): - process_case(adfobj, case_name, case_idx, var, odata, - seasons, pres_levs, plot_type, redo_plot, - vres, web_category) + #Loop over model cases: + for case_idx, case_name in enumerate(adfobj.data.case_names): + process_case(adfobj, case_name, case_idx, var, odata, + seasons, pres_levs, plot_type, redo_plot, + vres, web_category) @@ -194,8 +194,8 @@ def process_seasonal_data(mdata, odata, season, weight_season=True): mseason = pf.seasonal_mean(mdata, season=season, is_climo=True) oseason = pf.seasonal_mean(odata, season=season, is_climo=True) else: - mseason = mdata.sel(time=seasons[s]).mean(dim='time') - oseason = odata.sel(time=seasons[s]).mean(dim='time') + mseason = mdata.sel(time=season).mean(dim='time') + oseason = odata.sel(time=season).mean(dim='time') # Calculate differences dseason = mseason - oseason @@ -361,13 +361,14 @@ def check_existing_plots(adfobj, var, plot_loc, plot_type, case_name, redo_plot, "LatLon") return doplot + def process_2d_plots(adfobj, mdata, odata, case_name, case_nickname, var, seasons, plot_loc, plot_type, doplot, mseasons, oseasons, dseasons, pseasons, syear_case, eyear_case, syear_baseline, eyear_baseline, web_category, vres): """Process and generate 2D plots.""" - for s in seasons: + for s in seasons.keys(): plot_name = plot_loc / f"{var}_{s}_LatLon_Mean.{plot_type}" if doplot[plot_name] is None: continue @@ -402,7 +403,7 @@ def process_3d_plots(adfobj, mdata, odata, case_name, case_nickname, f"ref: {pres in odata['lev']}], so skipping.") continue - for s in seasons: + for s in seasons.keys(): plot_name = plot_loc / f"{var}_{pres}hpa_{s}_LatLon_Mean.{plot_type}" if doplot[plot_name] is None: continue From 60d90934cc3adb1ce255cd7dc76768ecde1b973e Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 14 Mar 2025 14:53:49 -0600 Subject: [PATCH 20/91] changed import for aod_latlon --- scripts/plotting/global_latlon_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plotting/global_latlon_map.py b/scripts/plotting/global_latlon_map.py index a2ade38c5..b69f40589 100644 --- a/scripts/plotting/global_latlon_map.py +++ b/scripts/plotting/global_latlon_map.py @@ -18,7 +18,7 @@ # Import local modules: import plotting_functions as pf -from .aod_latlon import aod_latlon +from aod_latlon import aod_latlon # Format warning messages: def my_formatwarning(msg, *args, **kwargs): From 25cf33cc107091944dcc05c7418aa43cfcc18998 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 14 Mar 2025 14:57:50 -0600 Subject: [PATCH 21/91] fixed defining dictionary in dataclass in aod_latlon.py --- scripts/plotting/aod_latlon.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/plotting/aod_latlon.py b/scripts/plotting/aod_latlon.py index 1c8c59bf8..6e536a4be 100644 --- a/scripts/plotting/aod_latlon.py +++ b/scripts/plotting/aod_latlon.py @@ -15,13 +15,18 @@ import plotting_functions as pf -from dataclasses import dataclass +from dataclasses import dataclass, field @dataclass class AODPlotConfig: """Configuration for AOD plots.""" seasons: list = ('DJF', 'MAM', 'JJA', 'SON') - season_names: dict = {'DJF': 'Dec-Jan-Feb', 'MAM': 'Mar-Apr-May', 'JJA': 'Jun-Jul-Aug', 'SON': 'Sep-Oct-Nov'} + season_names: dict = field(default_factory=lambda: { + 'DJF': 'Dec-Jan-Feb', + 'MAM': 'Mar-Apr-May', + 'JJA': 'Jun-Jul-Aug', + 'SON': 'Sep-Oct-Nov' + }) obs_sources: list = ('TERRA MODIS', 'MERRA2') var_name: str = 'AODVISdn' From 9e44da44088da4ee96762d2aba4129985498e268 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 14 Mar 2025 15:06:29 -0600 Subject: [PATCH 22/91] debugging loading reference case for aod plots --- scripts/plotting/aod_latlon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/plotting/aod_latlon.py b/scripts/plotting/aod_latlon.py index 6e536a4be..cfe5cfd49 100644 --- a/scripts/plotting/aod_latlon.py +++ b/scripts/plotting/aod_latlon.py @@ -132,6 +132,7 @@ def process_model_cases(adfobj, var, obs_data): # Process each case processed_data = [] for case_name in cases: + print(f"[process_model_cases] {case_name = }") # Load and process model data case_data = process_model_data(adfobj, case_name, var, ref_obs) if case_data is not None: From cb39ab3ffe7ea8f0676a0891c8886539dac0dccc Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 14 Mar 2025 15:09:25 -0600 Subject: [PATCH 23/91] debugging loading reference case: try logic to identify reference --- scripts/plotting/aod_latlon.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/plotting/aod_latlon.py b/scripts/plotting/aod_latlon.py index cfe5cfd49..35cd4e9ce 100644 --- a/scripts/plotting/aod_latlon.py +++ b/scripts/plotting/aod_latlon.py @@ -143,7 +143,10 @@ def process_model_cases(adfobj, var, obs_data): def process_model_data(adfobj, case_name, var, obs_shape): """Process model data and check grid compatibility.""" - ds_case = adfobj.data.load_climo_da(case_name, var) + if case_name == adfobj.data.ref_case_label: + ds_case = adfobj.data.load_reference_climo_da(case_name, var) + else: + ds_case = adfobj.data.load_climo_da(case_name, var) if ds_case is None: print(f"\t WARNING: No climo file for {case_name} variable {var}") return None From dba3435e02f2f2498917a9cfd0c17a62b18997c4 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 14 Mar 2025 15:23:08 -0600 Subject: [PATCH 24/91] debugging loading reference case: try logic to identify reference --- scripts/plotting/aod_latlon.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/plotting/aod_latlon.py b/scripts/plotting/aod_latlon.py index 35cd4e9ce..4b330ad0e 100644 --- a/scripts/plotting/aod_latlon.py +++ b/scripts/plotting/aod_latlon.py @@ -426,6 +426,7 @@ def aod_panel_latlon(adfobj, plot_titles, plot_params, data, season, obs_name, c extend_option = 'both' if symmetric else 'max' for ax, is_panel in [(axs[i], True), (ind_ax, False)]: + print(f"DEBUGGING: {type(ax) = }, {is_panel = } // {type(lon_mesh) = }, {lon_mesh.shape = } // {type(lat_mesh) = }, {lat_mesh.shape = } // {field_values.shape = }") img = ax.contourf(lon_mesh, lat_mesh, field_values, levels, cmap=cmap_option, extend=extend_option, transform=ccrs.PlateCarree()) From 945ef94bac1f24ff3389003622cd3f395f085c09 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 14 Mar 2025 15:32:20 -0600 Subject: [PATCH 25/91] debugging the plot generation --- scripts/plotting/aod_latlon.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/plotting/aod_latlon.py b/scripts/plotting/aod_latlon.py index 4b330ad0e..c1161436b 100644 --- a/scripts/plotting/aod_latlon.py +++ b/scripts/plotting/aod_latlon.py @@ -398,6 +398,7 @@ def aod_panel_latlon(adfobj, plot_titles, plot_params, data, season, obs_name, c # Generate each panel for i, field in enumerate(data): + print(f"DEBUGGING: {i = }, {field.shape = }") # Create individual plot ind_fig, ind_ax = plt.subplots(1, 1, figsize=((7*case_num)/2, 10/2), subplot_kw={'projection': ccrs.PlateCarree()}) @@ -408,7 +409,7 @@ def aod_panel_latlon(adfobj, plot_titles, plot_params, data, season, obs_name, c lat_values = field.lat.values field_values, lon_values = add_cyclic_point(field_values, coord=lon_values) lon_mesh, lat_mesh = np.meshgrid(lon_values, lat_values) - field_mean = np.nanmean(field_values) + field_mean = np.nanmean(field_values) ## THIS IS THE INCORRECT AVERAGE TO USE # Set plot parameters plot_param = plot_params[i] @@ -417,6 +418,8 @@ def aod_panel_latlon(adfobj, plot_titles, plot_params, data, season, obs_name, c if 'augment_levels' in plot_param: levels = sorted(np.append(levels, np.array(plot_param['augment_levels']))) + print(f"DEBUGGING: {levels = }") + plot_config = plot_titles[i] title = f"{plot_config['title']} Mean {field_mean:.2g}" From 9b7975f58ba4f0dae1c7deff8073b3f4bc80a7c9 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 14 Mar 2025 16:45:24 -0600 Subject: [PATCH 26/91] testing and debugging refactors... might be working --- scripts/plotting/aod_latlon.py | 24 +++++++------- scripts/plotting/global_latlon_map.py | 8 ++--- scripts/plotting/polar_map.py | 46 +++++++++++++-------------- 3 files changed, 38 insertions(+), 40 deletions(-) diff --git a/scripts/plotting/aod_latlon.py b/scripts/plotting/aod_latlon.py index c1161436b..c0c293c1f 100644 --- a/scripts/plotting/aod_latlon.py +++ b/scripts/plotting/aod_latlon.py @@ -132,7 +132,6 @@ def process_model_cases(adfobj, var, obs_data): # Process each case processed_data = [] for case_name in cases: - print(f"[process_model_cases] {case_name = }") # Load and process model data case_data = process_model_data(adfobj, case_name, var, ref_obs) if case_data is not None: @@ -397,19 +396,20 @@ def aod_panel_latlon(adfobj, plot_titles, plot_params, data, season, obs_name, c axs.append(plt.subplot(gs[case_num:, start:end], projection=ccrs.PlateCarree())) # Generate each panel - for i, field in enumerate(data): - print(f"DEBUGGING: {i = }, {field.shape = }") + for i, dataField in enumerate(data): # Create individual plot ind_fig, ind_ax = plt.subplots(1, 1, figsize=((7*case_num)/2, 10/2), subplot_kw={'projection': ccrs.PlateCarree()}) # Prepare data - field_values = field.values[:,:] - lon_values = field.lon.values - lat_values = field.lat.values - field_values, lon_values = add_cyclic_point(field_values, coord=lon_values) + # field_values = field.values[:,:] + # lon_values = field.lon.values + lat_values = dataField.lat + field_values, lon_values = add_cyclic_point(dataField, coord=dataField.lon) lon_mesh, lat_mesh = np.meshgrid(lon_values, lat_values) - field_mean = np.nanmean(field_values) ## THIS IS THE INCORRECT AVERAGE TO USE + + field_mean = np.nanmean(field_values) ## THIS IS PROBABLY THE INCORRECT AVERAGE TO USE + # field_mean = pf.spatial_average(dataField) # Set plot parameters plot_param = plot_params[i] @@ -418,8 +418,6 @@ def aod_panel_latlon(adfobj, plot_titles, plot_params, data, season, obs_name, c if 'augment_levels' in plot_param: levels = sorted(np.append(levels, np.array(plot_param['augment_levels']))) - print(f"DEBUGGING: {levels = }") - plot_config = plot_titles[i] title = f"{plot_config['title']} Mean {field_mean:.2g}" @@ -427,12 +425,12 @@ def aod_panel_latlon(adfobj, plot_titles, plot_params, data, season, obs_name, c cmap_option = (plot_param.get('colormap', plt.cm.bwr) if symmetric else plot_param.get('colormap', plt.cm.turbo)) extend_option = 'both' if symmetric else 'max' - + for ax, is_panel in [(axs[i], True), (ind_ax, False)]: - print(f"DEBUGGING: {type(ax) = }, {is_panel = } // {type(lon_mesh) = }, {lon_mesh.shape = } // {type(lat_mesh) = }, {lat_mesh.shape = } // {field_values.shape = }") img = ax.contourf(lon_mesh, lat_mesh, field_values, levels, cmap=cmap_option, extend=extend_option, - transform=ccrs.PlateCarree()) + transform=ccrs.PlateCarree(), + transform_first=True) ax.set_facecolor('gray') ax.coastlines() ax.set_title(title, fontsize=10) diff --git a/scripts/plotting/global_latlon_map.py b/scripts/plotting/global_latlon_map.py index b69f40589..8f1f3b06b 100644 --- a/scripts/plotting/global_latlon_map.py +++ b/scripts/plotting/global_latlon_map.py @@ -238,9 +238,9 @@ def plot_file_op(adfobj, plot_name, var, case_name, season, web_category, redo_p Returns ------- - int, None - Returns 1 if existing file is removed or no existing file. - Returns None if file exists and redo_plot is False + bool + Returns True if existing file is removed or no existing file, i.e. make the plot. + Returns False if file exists and redo_plot is False Notes ----- @@ -313,7 +313,7 @@ def process_plots(adfobj, mdata, odata, case_name, case_idx, var, seasons, case_name, seasons, pres_levs, has_dims, web_category, redo_plot) - if not any(value is None for value in doplot.values()): + if not any(value for value in doplot.values()): print(f"\t INFO: All plots exist for {var}. Redo is {redo_plot}. Existing plots added to website data.") return diff --git a/scripts/plotting/polar_map.py b/scripts/plotting/polar_map.py index bd9540c29..c99e5ac9a 100644 --- a/scripts/plotting/polar_map.py +++ b/scripts/plotting/polar_map.py @@ -126,28 +126,28 @@ def polar_map(adfobj): for s in seasons: for hemi_type in ["NHPolar", "SHPolar"]: - if pres_levs: - if has_lev: - for pres in pres_levs: - plot_name = plot_loc / f"{var}_{pres}hpa_{s}_{hemi_type}_Mean.{plot_type}" - info = { - 'path': plot_name, - 'var': f"{var}_{pres}hpa", - 'case': case_name, - 'case_idx': case_idx, - 'season': s, - 'type': hemi_type, - 'pressure': pres, - 'exists': plot_name.is_file() - } - plot_info.append(info) - if not (redo_plot or not info['exists']): - adfobj.add_website_data(info['path'], info['var'], - info['case'], category=web_category, - season=s, plot_type=hemi_type) - else: - all_plots_exist = False - else: + if pres_levs and has_lev: # 3-D variable & pressure levels specified + print(f"POLAR: {pres_levs = }") + for pres in pres_levs: + plot_name = plot_loc / f"{var}_{pres}hpa_{s}_{hemi_type}_Mean.{plot_type}" + info = { + 'path': plot_name, + 'var': f"{var}_{pres}hpa", + 'case': case_name, + 'case_idx': case_idx, + 'season': s, + 'type': hemi_type, + 'pressure': pres, + 'exists': plot_name.is_file() + } + plot_info.append(info) + if (redo_plot is False) and info['exists']: + adfobj.add_website_data(info['path'], info['var'], + info['case'], category=web_category, + season=s, plot_type=hemi_type) + else: + all_plots_exist = False + elif (not has_lev): # 2-D variable plot_name = plot_loc / f"{var}_{s}_{hemi_type}_Mean.{plot_type}" info = { 'path': plot_name, @@ -159,7 +159,7 @@ def polar_map(adfobj): 'exists': plot_name.is_file() } plot_info.append(info) - if not (redo_plot or not info['exists']): + if (redo_plot is False) and info['exists']: adfobj.add_website_data(info['path'], info['var'], info['case'], category=web_category, season=s, plot_type=hemi_type) From 594fb059e59a0db7d722a29b6e08dff99aa8f873 Mon Sep 17 00:00:00 2001 From: He Yanchun Date: Fri, 28 Mar 2025 13:23:53 +0100 Subject: [PATCH 27/91] add requirement for xesmf in relation to commit 17524b6 --- env/conda_environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/env/conda_environment.yaml b/env/conda_environment.yaml index 4729a9a1d..1ff0c70a4 100644 --- a/env/conda_environment.yaml +++ b/env/conda_environment.yaml @@ -14,4 +14,5 @@ dependencies: - xskillscore=0.0.5 - geocat-comp=2022.08.0 - python=3.11 + - xesm=0.8.7 prefix: /glade/work/$USER/conda-envs/adf_v0.11 From bf158c05f28a6228cd7bed23d27f607bcf586707 Mon Sep 17 00:00:00 2001 From: Meg Fowler Date: Mon, 7 Apr 2025 13:28:43 -0600 Subject: [PATCH 28/91] Add ENSO analysis and plot scripts --- config_cam_baseline_example_testENSO.yaml | 537 ++++++++++++++++++++++ lib/adf_variable_defaults.yaml | 2 +- scripts/analysis/ENSO_acrossRuns.py | 454 ++++++++++++++++++ scripts/plotting/enso_comparison_plots.py | 406 ++++++++++++++++ 4 files changed, 1398 insertions(+), 1 deletion(-) create mode 100644 config_cam_baseline_example_testENSO.yaml create mode 100644 scripts/analysis/ENSO_acrossRuns.py create mode 100644 scripts/plotting/enso_comparison_plots.py diff --git a/config_cam_baseline_example_testENSO.yaml b/config_cam_baseline_example_testENSO.yaml new file mode 100644 index 000000000..8113000ec --- /dev/null +++ b/config_cam_baseline_example_testENSO.yaml @@ -0,0 +1,537 @@ +#============================== +#config_cam_baseline_example.yaml + +#This is the main CAM diagnostics config file +#for doing comparisons of a CAM run against +#another CAM run, or a CAM baseline simulation. + +#Currently, if one is on NCAR's Casper or +#Cheyenne machine, then only the diagnostic output +#paths are needed, at least to perform a quick test +#run (these are indicated with "MUST EDIT" comments). +#Running these diagnostics on a different machine, +#or with a different, non-example simulation, will +#require additional modifications. +# +#Config file Keywords: +#-------------------- +# +#1. Using ${xxx} will substitute that text with the +# variable referenced by xxx. For example: +# +# cam_case_name: cool_run +# cam_climo_loc: /some/where/${cam_case_name} +# +# will set "cam_climo_loc" in the diagnostics package to: +# /some/where/cool_run +# +# Please note that currently this will only work if the +# variable only exists in one location in the file. +# +#2. Using ${.xxx} will do the same as +# keyword 1 above, but specifies which sub-section the +# variable is coming from, which is necessary for variables +# that are repeated in different subsections. For example: +# +# diag_basic_info: +# cam_climo_loc: /some/where/${diag_cam_climo.start_year} +# +# diag_cam_climo: +# start_year: 1850 +# +# will set "cam_climo_loc" in the diagnostics package to: +# /some/where/1850 +# +#Finally, please note that for both 1 and 2 the keywords must be lowercase. +#This is because future developments will hopefully use other keywords +#that are uppercase. Also please avoid using periods (".") in variable +#names, as this will likely cause issues with the current file parsing +#system. +#-------------------- +# +##============================== +# +# This file doesn't (yet) read environment variables, so the user must +# set this themselves. It is also a good idea to search the doc for 'user' +# to see what default paths are being set for output/working files. +# +# Note that the string 'USER-NAME-NOT-SET' is used in the jupyter script +# to check for a failure to customize +# +user: 'mdfowler' + + +#This first set of variables specify basic info used by all diagnostic runs: +diag_basic_info: + + #Is this a model vs observations comparison? + #If "false" or missing, then a model-model comparison is assumed: + compare_obs: false + + #Generate HTML website (assumed false if missing): + #Note: The website files themselves will be located in the path + #specified by "cam_diag_plot_loc", under the "/website" subdirectory, + #where "" is the subdirectory created for this particular diagnostics run + #(usually "case_vs_obs_XXX" or "case_vs_baseline_XXX"). + create_html: true + + #Location of observational datasets: + #Note: this only matters if "compare_obs" is true and the path + #isn't specified in the variable defaults file. + obs_data_loc: /glade/campaign/cgd/amp/amwg/ADF_obs + + #Location where re-gridded and interpolated CAM climatology files are stored: + cam_regrid_loc: /glade/derecho/scratch/${user}/ADF/regrid + + #Overwrite CAM re-gridded files? + #If false, or missing, then regridding will be skipped for regridded variables + #that already exist in "cam_regrid_loc": + cam_overwrite_regrid: false + + #Location where diagnostic plots are stored: + cam_diag_plot_loc: /glade/derecho/scratch/${user}/ADF/plots + + #Location of ADF variable plotting defaults YAML file: + #If left blank or missing, ADF/lib/adf_variable_defaults.yaml will be used + #Uncomment and change path for custom variable defaults file + #defaults_file: /some/path/to/defaults/file.yaml + + #Vertical pressure levels (in hPa) on which to plot 3-D variables + #when using horizontal (e.g. lat/lon) map projections. + #If this config option is missing, then no 3-D variables will be plotted on + #horizontal maps. Please note too that pressure levels must currently match + #what is available in the observations file in order to be plotted in a + #model vs obs run: + plot_press_levels: [200,850] + + #Longitude line on which to center all lat/lon maps. + #If this config option is missing then the central + #longitude will default to 180 degrees E. + central_longitude: 180 + + #Number of processors on which to run the ADF. + #If this config variable isn't present then + #the ADF defaults to one processor. Also, if + #you set it to "*" then it will default + #to all of the processors available on a + #single node/machine: + num_procs: 8 + + #If set to true, then redo all plots even if they already exist. + #If set to false, then if a plot is found it will be skipped: + redo_plot: true + +#This second set of variables provides info for the CAM simulation(s) being diagnosed: +diag_cam_climo: + + # History file list of strings to match + # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] + # Only affects timeseries as everything else uses the created timeseries + # Default: + hist_str: cam.h0a + + #Calculate climatologies? + #If false, the climatology files will not be created: + calc_cam_climo: true + + #Overwrite CAM climatology files? + #If false, or not prsent, then already existing climatology files will be skipped: + cam_overwrite_climo: false + + #Name of CAM case (or CAM run name): + cam_case_name: b.e30_alpha06b.B1850C_LTso.ne30_t232_wgx3.132 + + #Case nickname + #NOTE: if nickname starts with '0' - nickname must be in quotes! + # ie '026a' as opposed to 026a + #If missing or left blank, will default to cam_case_name + case_nickname: '132' + + #Location of CAM history (h0) files: + #Example test files + # cam_hist_loc: /glade/campaign/cgd/amp/amwg/ADF_test_cases/${diag_cam_climo.cam_case_name} + cam_hist_loc: /glade/derecho/scratch/hannay/archive//b.e30_alpha06b.B1850C_LTso.ne30_t232_wgx3.132/atm/hist + + #Location of CAM climatologies (to be created and then used by this script) + cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/climo + + #model year when time series files should start: + #Note: Leaving this entry blank will make time series + # start at earliest available year. + start_year: 2 + + #model year when time series files should end: + #Note: Leaving this entry blank will make time series + # end at latest available year. + end_year: 44 + + #Do time series files exist? + #If True, then diagnostics assumes that model files are already time series. + #If False, or if simply not present, then diagnostics will attempt to create + #time series files from history (time-slice) files: + cam_ts_done: false + + #Save interim time series files? + #WARNING: This can take up a significant amount of space, + # but will save processing time the next time + cam_ts_save: true + + #Overwrite time series files, if found? + #If set to false, then time series creation will be skipped if files are found: + cam_overwrite_ts: false + + #Location where time series files are (or will be) stored: + cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/ts + + #TEM diagnostics + #--------------- + #TEM history file number + #If missing or blank, ADF will default to h4 + tem_hist_str: cam.h4 + + #Location where TEM files are stored: + #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! + cam_tem_loc: /glade/derecho/scratch/${user}/${diag_cam_climo.cam_case_name}/tem/ + + #Overwrite TEM files, if found? + #If set to false, then TEM creation will be skipped if files are found: + overwrite_tem: false + + #---------------------- + + #You can alternatively provide a list of cases, which will make the ADF + #apply the same diagnostics to each case separately in a single ADF session. + #All of the config variables below show how it is done, and are the only ones + #that need to be lists. This also automatically enables the generation of + #a "main_website" in "cam_diag_plot_loc" that brings all of the different cases + #together under a single website. + + #Also please note that config keywords cannot currently be used in list mode. + + #cam_case_name: + # - b.e23_alpha17f.BLT1850.ne30_t232.098 + # - b.e23_alpha17f.BLT1850.ne30_t232.095 + + #Case nickname + #NOTE: if nickname starts with '0' - nickname must be in quotes! + # ie '026a' as opposed to 026a + #If missing or left blank, will default to cam_case_name + #case_nickname: + # - cool nickname + # - cool nickname 2 + + #calc_cam_climo: + # - true + # - true + + #cam_overwrite_climo: + # - false + # - false + + #cam_hist_loc: + # - /glade/campaign/cgd/amp/amwg/ADF_test_cases/b.e23_alpha17f.BLT1850.ne30_t232.098 + # - /glade/campaign/cgd/amp/amwg/ADF_test_cases/b.e23_alpha17f.BLT1850.ne30_t232.095 + + #cam_climo_loc: + # - /some/where/you/want/to/have/climo_files/ #MUST EDIT! + # - /the/same/or/some/other/climo/files/location + + #start_year: + # - 10 + # - 10 + + #end_year: + # - 14 + # - 14 + + #cam_ts_done: + # - false + # - false + + #cam_ts_save: + # - true + # - true + + #cam_overwrite_ts: + # - false + # - false + + #cam_ts_loc: + # - /some/where/you/want/to/have/time_series_files + # - /same/or/different/place/you/want/files + + #TEM diagnostics + #--------------- + #TEM history file number + #If missing or blank, ADF will default to h4 + #tem_hist_str: + # - cam.h4 + # - cam.h# + + #Location where TEM files are stored: + #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! + #cam_tem_loc: + # - /some/where/you/want/to/have/TEM_files/ + # - /same/or/different/place/you/want/TEM_files/ + + #Overwrite TEM files, if found? + #If set to false, then TEM creation will be skipped if files are found: + #overwrite_tem: + # - false + # - true + + #---------------------- + + +#This third set of variables provide info for the CAM baseline climatologies. +#This only matters if "compare_obs" is false: +diag_cam_baseline_climo: + + # History file list of strings to match + # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] + # Only affects timeseries as everything else uses the created timeseries + # Default: + hist_str: cam.h0a + + #Calculate cam baseline climatologies? + #If false, the climatology files will not be created: + calc_cam_climo: true + + #Overwrite CAM climatology files? + #If false, or not present, then already existing climatology files will be skipped: + cam_overwrite_climo: false + + #Name of CAM baseline case: + cam_case_name: b.e23_alpha17f.BLT1850.ne30_t232.093 + + #Baseline case nickname + #NOTE: if nickname starts with '0' - nickname must be in quotes! + # ie '026a' as opposed to 026a + #If missing or left blank, will default to cam_case_name + case_nickname: #cool nickname + + #Location of CAM baseline history (h0) files: + #Example test files + cam_hist_loc: /glade/campaign/cgd/amp/amwg/ADF_test_cases/${diag_cam_baseline_climo.cam_case_name} + + #Location of baseline CAM climatologies: + cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/climo + + #model year when time series files should start: + #Note: Leaving this entry blank will make time series + # start at earliest available year. + start_year: 10 + + #model year when time series files should end: + #Note: Leaving this entry blank will make time series + # end at latest available year. + end_year: 14 + + #Do time series files need to be generated? + #If True, then diagnostics assumes that model files are already time series. + #If False, or if simply not present, then diagnostics will attempt to create + #time series files from history (time-slice) files: + cam_ts_done: false + + #Save interim time series files for baseline run? + #WARNING: This can take up a significant amount of space: + cam_ts_save: true + + #Overwrite baseline time series files, if found? + #If set to false, then time series creation will be skipped if files are found: + cam_overwrite_ts: false + + #Location where time series files are (or will be) stored: + cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/ts + + #TEM diagnostics + #--------------- + #TEM history file number + #If missing or blank, ADF will default to h4 + tem_hist_str: cam.h4 + + #Location where TEM files are stored: + #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! + cam_tem_loc: /glade/derecho/scratch/${user}/${diag_cam_baseline_climo.cam_case_name}/tem/ + + #Overwrite TEM files, if found? + #If set to false, then TEM creation will be skipped if files are found: + overwrite_tem: false + + +#This fourth set of variables provides settings for calling the Climate Variability +# Diagnostics Package (CVDP). If cvdp_run is set to true the CVDP will be set up and +# run in background mode, likely completing after the ADF has completed. +# If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed +# in the diag_var_list variable listing. +# For more CVDP information: https://www.cesm.ucar.edu/working_groups/CVC/cvdp/ +diag_cvdp_info: + + # Run the CVDP on the listed run(s)? + cvdp_run: false + + # CVDP code path, sets the location of the CVDP codebase + # CGD systems path = /home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ + # CISL systems path = /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ + # github location = https://github.com/NCAR/CVDP-ncl + cvdp_codebase_loc: /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ + + # Location where cvdp codebase will be copied to and diagnostic plots will be stored + cvdp_loc: /glade/derecho/scratch/${user}/ADF/cvdp/ + + # tar up CVDP results? + cvdp_tar: false + +# This set of variables provides settings for calling NOAA's +# Model Diagnostic Task Force (MDTF) diagnostic package. +# https://github.com/NOAA-GFDL/MDTF-diagnostics +# +# If mdtf_run: true, the MDTF will be set up and +# run in background mode, likely completing after the ADF has completed. +# +# WARNING: This currently only runs on CASPER (not derecho) +# +# The variables required depend on the diagnostics (PODs) selected. +# AMWG-developed PODS and their required variables: +# (Note that PRECT can be computed from PRECC & PRECL) +# - MJO_suite: daily PRECT, FLUT, U850, U200, V200 (all required) +# - Wheeler-Kiladis Wavenumber Frequency Spectra: daily PRECT, FLUT, U200, U850, OMEGA500 +# (will use what is available) +# - Blocking (Rich Neale): daily OMEGA500 +# - Precip Diurnal Cycle (Rich Neale): 3-hrly PRECT +# +# Many other diagnostics are available; see +# https://mdtf-diagnostics.readthedocs.io/en/main/sphinx/start_overview.html + +# +diag_mdtf_info: + # Run the MDTF on the model cases + mdtf_run: false + + # The file that will be written by ADF to input to MDTF. Call this whatever you want. + mdtf_input_settings_filename : mdtf_input.json + + ## MDTF code path, sets the location of the MDTF codebase and pre-compiled conda envs + # CHANGE if you have any: your own MDTF code, installed conda envs and/or obs_data + + mdtf_codebase_path : /glade/campaign/cgd/amp/amwg/mdtf + mdtf_codebase_loc : ${mdtf_codebase_path}/MDTF-diagnostics.v3.1.20230817.ADF + conda_root : /glade/u/apps/opt/conda + conda_env_root : ${mdtf_codebase_path}/miniconda2/envs.MDTFv3.1.20230412/ + OBS_DATA_ROOT : ${mdtf_codebase_path}/obs_data + + # SET this to a writable dir. The ADF will place ts files here for the MDTF to read (adds the casename) + MODEL_DATA_ROOT : ${diag_cam_climo.cam_ts_loc}/mdtf/inputdata/model + + # Choose diagnostics (PODs). Full list of available PODs: https://github.com/NOAA-GFDL/MDTF-diagnostics + pod_list : [ "MJO_suite" ] + + # Intermediate/output file settings + make_variab_tar: false # tar up MDTF results + save_ps : false # save postscript figures in addition to bitmaps + save_nc : false # save netCDF files of processed data (recommend true when starting with new model data) + overwrite: true # overwrite results in OUTPUT_DIR; otherwise results will be saved under a unique name + + # Settings used in debugging: + verbose : 3 # Log verbosity level. + test_mode: false # Set to true for framework test. Data is fetched but PODs are not run. + dry_run : false # Framework test. No external commands are run and no remote data is copied. Implies test_mode. + + # Settings that shouldn't change in ADF implementation for now + data_type : single_run # single_run or multi_run (only works with single right now) + data_manager : Local_File # Fetch data or it is local? + environment_manager : Conda # Manage dependencies + + + +#+++++++++++++++++++++++++++++++++++++++++++++++++++ +#These variables below only matter if you are using +#a non-standard method, or are adding your own +#diagnostic scripts. +#+++++++++++++++++++++++++++++++++++++++++++++++++++ + +#Note: If you want to pass arguments to a particular script, you can +#do it like so (using the "averaging_example" script in this case): +# - {create_climo_files: {kwargs: {clobber: true}}} + +#Name of time-averaging scripts being used to generate climatologies. +#These scripts must be located in "scripts/averaging": +time_averaging_scripts: + - create_climo_files + #- create_TEM_files #To generate TEM files, please un-comment + +#Name of regridding scripts being used. +#These scripts must be located in "scripts/regridding": +regridding_scripts: + - regrid_and_vert_interp + +#List of analysis scripts being used. +#These scripts must be located in "scripts/analysis": +analysis_scripts: + - amwg_table + - ENSO_acrossRuns + #- aerosol_gas_tables + +#List of plotting scripts being used. +#These scripts must be located in "scripts/plotting": +plotting_scripts: + - global_latlon_map + # - global_latlon_vect_map + # - zonal_mean + # - meridional_mean + # - polar_map + # - cam_taylor_diagram + # - qbo + # - ozone_diagnostics + - enso_comparison_plots + #- tape_recorder + #- tem + #- regional_map_multicase #To use this please un-comment and fill-out + #the "region_multicase" section below + +#List of CAM variables that will be processesd: +#If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed +diag_var_list: + - SWCF + - LWCF + - PRECC + - PRECL + - PSL + - Q + - U + - T + - RELHUM + - TREFHT + - TS + - TAUX + - TAUY + - FSNT + - FLNT + - LANDFRAC + - O3 + +# +# MDTF recommended variables +# - FLUT +# - OMEGA500 +# - PRECT +# - PS +# - PSL +# - U200 +# - U850 +# - V200 +# - V850 + +# Options for multi-case regional contour plots (./plotting/regional_map_multicase.py) +# region_multicase: +# region_spec: [slat, nlat, wlon, elon] +# region_time_option: # If calendar, will look for specified years. If zeroanchor will use a nyears starting from year_offset from the beginning of timeseries +# region_start_year: +# region_end_year: +# region_nyear: +# region_year_offset: +# region_month: +# region_season: +# region_variables: + +#END OF FILE diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index 60743359c..ddba67426 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -69,7 +69,7 @@ # Available ADF Default Plot Types #+++++++++++++ default_ptypes: ["Tables","LatLon","LatLon_Vector","Zonal","Meridional", - "NHPolar","SHPolar","TimeSeries","Special"] + "NHPolar","SHPolar","TimeSeries","ENSO","Special",] #+++++++++++++ # Constants diff --git a/scripts/analysis/ENSO_acrossRuns.py b/scripts/analysis/ENSO_acrossRuns.py new file mode 100644 index 000000000..09629cab1 --- /dev/null +++ b/scripts/analysis/ENSO_acrossRuns.py @@ -0,0 +1,454 @@ +import os +import glob +import numpy as np +import xarray as xr +import pandas as pd +import datetime +from datetime import date, timedelta +import scipy.stats as stats +import scipy.signal as signal +import matplotlib +import matplotlib.pyplot as plt +import matplotlib.patches as patches +import matplotlib.ticker as mticker +import cartopy +import cartopy.crs as ccrs + +# Import necessary ADF modules: +from pathlib import Path +from adf_base import AdfError + +## Functions ## + + +def ENSO_acrossRuns(adf, clobber=False, search=None): + """ + This function computes ENSO statistics for new cases + + Description of needed inputs from ADF: + + case_name -> Name of CAM case provided by "cam_case_name" + input_ts_loc -> Location of CAM time series files provided by "cam_ts_loc" + output_loc -> Location to write new dev file to, provided by "cam_climo_loc" + var_list -> List of CAM output variables provided by "diag_var_list" + + """ + ## Define some basics + lat_n = 10.0 + lat_s = -10.0 + + # Nino3.4 + lat_n34 = 5 + lat_s34 = -5 + lon_e34 = 190 + lon_w34 = 240 + + # Nino3 + lat_n3 = 5 + lat_s3 = -5 + lon_e3 = 210 + lon_w3 = 270 + + # Nino 4 + lat_n4 = 5 + lat_s4 = -5 + lon_e4 = 160 + lon_w4 = 210 + + # Nino 1+2 + lat_n12 = 0 + lat_s12 = -10 + lon_e12 = 270 + lon_w12 = 280 + + ## Read in ENSO characteristics from obs file + obs_ds = xr.open_dataset('/glade/derecho/scratch/mdfowler/ENSOmetrics_Obs.nc') + + + ## Read in data for new case + # - - - - - - - - - - - - - - - - - - - - - - - + case_names = adf.get_cam_info("cam_case_name", required=True) + input_ts_locs = adf.get_cam_info("cam_ts_loc", required=True) + #Extract simulation years: + start_year = adf.climo_yrs["syears"] + end_year = adf.climo_yrs["eyears"] + + + #Loop over CAM cases: + for case_idx, case_name in enumerate(case_names): + #Create "Path" objects: + input_location = Path(input_ts_locs[case_idx]) + + case_shortName = adf.get_cam_info("case_nickname", required=True)[case_idx] + + #Check that time series input directory actually exists: + if not input_location.is_dir(): + errmsg = f"Time series directory '{input_ts_locs}' not found. Script is exiting." + raise AdfError(errmsg) + + #Check model year bounds: + syr, eyr = check_averaging_interval(start_year[case_idx], end_year[case_idx]) + + ts = adf.data.load_timeseries_da(case_name, 'TS') + landfrac = adf.data.load_timeseries_da(case_name, 'LANDFRAC') + + ts = ts.assign_coords({"case": case_shortName}) + + #Extract data subset using provided year bounds: + tslice = get_time_slice_by_year(ts.time, int(syr), int(eyr)) + ts = ts.isel(time=tslice).sel(lat=slice(-10,10)) + landfrac = landfrac.isel(time=tslice).sel(lat=slice(-10,10)) + + # - - - - - - - - - - - - - - - - - - - - - - - + # Start computing ENSO-relevant pieces + # - - - - - - - - - - - - - - - - - - - - - - - + + # Use ocean points only + ocnMask = landfrac.values + ocnMask[ocnMask>0.45] = np.nan + ocnMask[ocnMask<=0.45] = 1 + + # Detrend data + TS = ts + ts_detrend = signal.detrend(TS, axis=0, type='linear') + # Get SST + sst = ts_detrend * ocnMask + sst_raw_data = TS.values * ocnMask + + sst = xr.DataArray(sst, + coords={'time': ts.time.values, + 'lat': ts.lat.values, + 'lon': ts.lon.values}, + dims=["time", "lat", "lon"]) + + # Remove annual cycle from monthly data + sst_anom = rmMonAnnCyc(sst) + + ## Compute nino 3.4 index + ## - - - - - - - - - - - - + ilats = np.where((sst_anom.lat.values>=lat_s34) & (sst_anom.lat.values<=lat_n34))[0] + ilons = np.where((sst_anom.lon.values>=lon_e34) & (sst_anom.lon.values<=lon_w34))[0] + + regionTS = sst_anom.isel(lat=ilats, lon=ilons) + coswgt = np.cos(np.deg2rad(regionTS.lat)) + nino34 = regionTS.weighted(coswgt).mean(('lon','lat')) + del ilats,ilons + + ## Compute nino 3 index + ## - - - - - - - - - - - - + ilats = np.where((sst_anom.lat.values>=lat_s3) & (sst_anom.lat.values<=lat_n3))[0] + ilons = np.where((sst_anom.lon.values>=lon_e3) & (sst_anom.lon.values<=lon_w3))[0] + + regionTS = sst_anom.isel(lat=ilats, lon=ilons) + coswgt = np.cos(np.deg2rad(regionTS.lat)) + nino3 = regionTS.weighted(coswgt).mean(('lon','lat')) + del ilats,ilons + + ## Compute nino 4 index + ## - - - - - - - - - - - - + ilats = np.where((sst_anom.lat.values>=lat_s4) & (sst_anom.lat.values<=lat_n4))[0] + ilons = np.where((sst_anom.lon.values>=lon_e4) & (sst_anom.lon.values<=lon_w4))[0] + + regionTS = sst_anom.isel(lat=ilats, lon=ilons) + coswgt = np.cos(np.deg2rad(regionTS.lat)) + nino4 = regionTS.weighted(coswgt).mean(('lon','lat')) + del ilats,ilons + + ## Compute nino 1.2 index + ## - - - - - - - - - - - - + ilats = np.where((sst_anom.lat.values>=lat_s12) & (sst_anom.lat.values<=lat_n12))[0] + ilons = np.where((sst_anom.lon.values>=lon_e12) & (sst_anom.lon.values<=lon_w12))[0] + + regionTS = sst_anom.isel(lat=ilats, lon=ilons) + coswgt = np.cos(np.deg2rad(regionTS.lat)) + nino12 = regionTS.weighted(coswgt).mean(('lon','lat')) + del ilats,ilons + + ## Western extent of SST correlation + ## - - - - - - - - - - - - - - - - - - + + cor_case = getLagCorr(0, nino34, sst_anom) + + ## Figure out western-most longitude of zero contour in pacific + # This works by creating a plot with a contour and identifying the western point, but I'm closing that plot + fig,axs = plt.subplots(1,1,figsize=(20,7),subplot_kw={"projection":ccrs.PlateCarree(central_longitude=210)}) + + lon0, lat0 = getWesternPoint(cor_case, 0) + lon0p5, lat0p5 = getWesternPoint(cor_case, 0.5) + + plt.close() + + ## Variance and Auto-correlation + ## - - - - - - - - - - - - - - - - + + nino34_var = np.full([12], np.nan) + nino34_ac = np.full([48], np.nan) + transit_month = np.nan + + nino3_var = np.full([12], np.nan) + nino3_ac = np.full([48], np.nan) + transit_month3 = np.nan + + nino4_var = np.full([12], np.nan) + nino4_ac = np.full([48], np.nan) + transit_month4 = np.nan + + nino12_var = np.full([12], np.nan) + nino12_ac = np.full([48], np.nan) + transit_month12 = np.nan + + + ## Reproduce the variance plot from Rich + # NOTE: doesn't match exactly, likely because I use cos(lat) for weights instead of a gaussian like Rich + nino34_var = nino34.groupby('time.month').var() + nino3_var = nino3.groupby('time.month').var() + nino4_var = nino4.groupby('time.month').var() + nino12_var = nino12.groupby('time.month').var() + + + ## Reproduce the autocorrelation plot from Rich + nino34_pd = pd.Series(nino34.values) + nino3_pd = pd.Series(nino3.values) + nino4_pd = pd.Series(nino4.values) + nino12_pd = pd.Series(nino12.values) + + for iLag in range(48): + nino34_ac[iLag] = nino34_pd.autocorr(lag=iLag) + if ((nino34_ac[iLag-1]>0) & (nino34_ac[iLag]<0) & (np.isfinite(transit_month)==False)): + transit_month = iLag-1 + + nino3_ac[iLag] = nino3_pd.autocorr(lag=iLag) + if ((nino3_ac[iLag-1]>0) & (nino3_ac[iLag]<0) & (np.isfinite(transit_month3)==False)): + transit_month3 = iLag-1 + + nino4_ac[iLag] = nino4_pd.autocorr(lag=iLag) + if ((nino4_ac[iLag-1]>0) & (nino4_ac[iLag]<0) & (np.isfinite(transit_month4)==False)): + transit_month4 = iLag-1 + + nino12_ac[iLag] = nino12_pd.autocorr(lag=iLag) + if ((nino12_ac[iLag-1]>0) & (nino12_ac[iLag]<0) & (np.isfinite(transit_month12)==False)): + transit_month12 = iLag-1 + + + ## SST biases + ## - - - - - - - - - - - - - - - - + sst_bias = np.full([ 4, len(ts.lat.values), len(ts.lon.values)], np.nan) + sst_raw = np.full([ 4, len(ts.lat.values), len(ts.lon.values)], np.nan) + sst_anom = np.full([ 4, len(ts.lat.values), len(ts.lon.values)], np.nan) + + sst_raw_data = xr.DataArray(sst_raw_data, + coords={'time': ts.time.values, + 'lat': ts.lat.values, + 'lon': ts.lon.values}, + dims=["time", "lat", "lon"]) + + # Get seasonal means + month_length = sst.time.dt.days_in_month + weights = ( month_length.groupby("time.season") / month_length.groupby("time.season").sum() ) + # Calculate the weighted average + sst_case_weighted = (sst * weights).groupby("time.season").sum(dim="time") + sst_raw_case_weighted = (sst_raw_data * weights).groupby("time.season").sum(dim="time") + del month_length,weights + + # Compute bias vs. observations (seasonally) + sst_bias[:,:,:] = (sst_case_weighted - obs_ds.sst_obs_weighted) * ocnMask[0,:,:] + sst_raw[:,:,:] = (sst_raw_case_weighted) * ocnMask[0,:,:] + sst_anom[:,:,:] = sst_case_weighted * ocnMask[0,:,:] + + sst_bias = xr.DataArray(sst_bias, + coords={ + 'season': sst_case_weighted.season.values, + 'lat': sst_case_weighted.lat.values, + 'lon':sst_case_weighted.lon.values}, + dims=["season", "lat", "lon"]) + + sst_raw = xr.DataArray(sst_raw, + coords={ + 'season': sst_case_weighted.season.values, + 'lat': sst_case_weighted.lat.values, + 'lon':sst_case_weighted.lon.values}, + dims=["season", "lat", "lon"]) + + sst_anom = xr.DataArray(sst_anom, + coords={ + 'season': sst_case_weighted.season.values, + 'lat': sst_case_weighted.lat.values, + 'lon':sst_case_weighted.lon.values}, + dims=["season", "lat", "lon"]) + + ## Get the longitude of the "cold tongue", approximated by the contour of the 299 K SST line + # This works by creating a plot with a contour and identifying the western point, but I'm closing that plot + fig,axs = plt.subplots(1,1,figsize=(20,7),subplot_kw={"projection":ccrs.PlateCarree(central_longitude=210)}) + + lon299, lat299 = getWesternPoint(sst_raw.mean(dim='season'),299) + + plt.close() + + # Mean SST in nino3.4 region + ilats = np.where((sst_raw.lat.values>=lat_s34) & (sst_raw.lat.values<=lat_n34))[0] + ilons = np.where((sst_raw.lon.values>=lon_e34) & (sst_raw.lon.values<=lon_w34))[0] + + # # Compute weights and get Nino3.4 + regionTS = sst_raw.isel(lat=ilats, lon=ilons) + coswgt = np.cos(np.deg2rad(regionTS.lat)) + sst_raw_nino34 = regionTS.weighted(coswgt).mean(('lon','lat')) + + + # - - - - - - - - - - - - - - - - - - - - - - - + # Add ENSO stats for new case to existing DS + # - - - - - - - - - - - - - - - - - - - - - - - + ## Combine all of above into a single DS, as in dev_ds + + thisDev_DS = xr.Dataset( + data_vars = dict( + startYear = (['case'], [ts['time.year'].values[0]]), + endYear = (['case'], [ts['time.year'].values[-1]]), + nino34_zeroContour = (['case'], [lon0]), + nino34_0p5Contour = (['case'], [lon0p5]), + nino34_variance = (['case', 'nMonths_12'], [nino34_var]), + nino34_autocorr = (['case', 'nMonths_48'], [nino34_ac]), + nino34_transMonth = (['case'], [transit_month]), + + nino3_variance = (['case', 'nMonths_12'], [nino3_var]), + nino3_autocorr = (['case', 'nMonths_48'], [nino3_ac]), + nino3_transMonth = (['case'], [transit_month3]), + + nino4_variance = (['case', 'nMonths_12'], [nino4_var]), + nino4_autocorr = (['case', 'nMonths_48'], [nino4_ac]), + nino4_transMonth = (['case'], [transit_month4]), + + nino12_variance = (['case', 'nMonths_12'], [nino12_var]), + nino12_autocorr = (['case', 'nMonths_48'], [nino12_ac]), + nino12_transMonth = (['case'], [transit_month12]), + + sst_raw = (['case', 'season', 'lat', 'lon'], [sst_raw.values]), + sst_raw_nino34 = (['case', 'season'], [sst_raw_nino34.values]), + sst_bias = (['case', 'season', 'lat', 'lon'], [sst_bias.values]), + sst_anom = (['case', 'season', 'lat', 'lon'], [sst_anom.values]), + + sst_lon299 = (['case'], [lon299]), + + ), + + coords = dict( + case=([ts.case.values]), + nMonths_48=np.arange(48), + nMonths_12=np.arange(12), + season=sst_bias.season.values, + lat=sst_bias.lat.values, + lon=sst_bias.lon.values, + ) + ) + + dev_ds = xr.open_dataset("/glade/derecho/scratch/mdfowler/ENSOmetrics_CESM3dev.nc") + newDS = xr.concat([dev_ds, thisDev_DS], dim="case") + + ## Save that new DS out to dev_ds and reload + dev_ds.close() + + newDS.to_netcdf("/glade/derecho/scratch/mdfowler/ENSOmetrics_CESM3dev.nc", mode='w') + + + +def get_time_slice_by_year(time, startyear, endyear): + if not hasattr(time, 'dt'): + print("Warning: get_time_slice_by_year requires the `time` parameter to be an xarray time coordinate with a dt accessor. Returning generic slice (which will probably fail).") + return slice(startyear, endyear) + start_time_index = np.argwhere((time.dt.year >= startyear).values).flatten().min() + end_time_index = np.argwhere((time.dt.year <= endyear).values).flatten().max() + return slice(start_time_index, end_time_index+1) + + +## Calculate nino anomalies +def rmMonAnnCyc(DS): + + climatology = DS.groupby("time.month").mean("time") + anomalies = DS.groupby("time.month") - climatology + + return anomalies + +def getLagCorr(lag, nino34, corDS): + + if lag>0: + A = nino34[:-lag] + # B = sst_anom.isel(case=iCase).shift(time=lag).isel(time=slice(lag,len(sst_anom.time.values))) + # B['time'] = sst_anom.time.values[:-lag] # To get xr.corr to work, need to have the "same" time for computing corrs + B = corDS.shift(time=-lag).isel(time=slice(0,len(corDS.time.values)-lag)) + B['time'] = corDS.time.values[:-lag] # To get xr.corr to work, need to have the "same" time for computing corrs + elif lag<0: + A = nino34[-lag:] + A['time'] = nino34.time.values[:lag] + B = corDS.isel(time=slice(0,len(corDS.time.values)+lag)) + elif lag==0: + A = nino34 + B = corDS + + cor = xr.corr(A, B, dim="time") + + return cor + +def getWesternPoint(DS, contourLev): + corrs_sel = DS.sel(lon=slice(120,240), lat=slice(-10,10)) + c2 = plt.contour(corrs_sel.lon.values,corrs_sel.lat.values , corrs_sel, [contourLev], transform=ccrs.PlateCarree()) + + # Add contour marker + maybeLon = [] + maybeLat = [] + lenSeg = 0 + for iSegs in range(len(c2.allsegs[0])): + dat0 = c2.allsegs[0][iSegs] + western_most_lon = np.nanmin(dat0[:,0]) + iMatchLat = np.where(dat0[:,0]==western_most_lon)[0] + maybeLon = np.append(maybeLon, western_most_lon) + maybeLat = np.append(maybeLat, dat0[int(iMatchLat[0]),1]) + if len(dat0)>lenSeg: + lenSeg= len(dat0) + iselSeg = iSegs + + # axs.plot(maybeLon[iselSeg], maybeLat[iselSeg], 'o', color='limegreen', markersize=5, transform=ccrs.PlateCarree() ) + + return maybeLon[iselSeg],maybeLat[iselSeg] + +def check_averaging_interval(syear_in, eyear_in): + #For now, make sure year inputs are integers or None, + #in order to allow for the zero additions done below: + if syear_in: + check_syr = int(syear_in) + else: + check_syr = None + #end if + + if eyear_in: + check_eyr = int(eyear_in) + else: + check_eyr = None + + #Need to add zeros if year values aren't long enough: + #------------------ + #start year: + if check_syr: + assert check_syr >= 0, 'Sorry, values must be positive whole numbers.' + try: + syr = f"{check_syr:04d}" + except: + errmsg = " 'start_year' values must be positive whole numbers" + errmsg += f"not '{syear_in}'." + raise AdfError(errmsg) + else: + syr = None + #End if + + #end year: + if check_eyr: + assert check_eyr >= 0, 'Sorry, end_year values must be positive whole numbers.' + try: + eyr = f"{check_eyr:04d}" + except: + errmsg = " 'end_year' values must be positive whole numbers" + errmsg += f"not '{eyear_in}'." + raise AdfError(errmsg) + else: + eyr = None + #End if + return syr, eyr + diff --git a/scripts/plotting/enso_comparison_plots.py b/scripts/plotting/enso_comparison_plots.py new file mode 100644 index 000000000..aa60c1bc5 --- /dev/null +++ b/scripts/plotting/enso_comparison_plots.py @@ -0,0 +1,406 @@ +""" +Generate plots that compare ENSO characteristics across various versions of CESM development +""" + +import xarray as xr +import numpy as np +import matplotlib +import matplotlib.pyplot as plt +import matplotlib.patches as patches +import matplotlib.ticker as mticker +import cartopy +import cartopy.crs as ccrs +import warnings # use to warn user about missing files +from pathlib import Path + +def enso_comparison_plots(adfobj): + """ + This script/function is designed to generate ENSO-related plots across + various CESM simulations + + Parameters + ---------- + adfobj : AdfDiag + The diagnostics object that contains all the configuration information + + Returns + ------- + Does not return a value; produces plots and saves files. + """ + + # Notify user that script has started: + msg = "\n Generating ENSO plots to compare against all runs..." + print(f"{msg}\n {'-' * (len(msg)-3)}") + + plot_locations = adfobj.plot_location + plot_type = adfobj.get_basic_info('plot_type') + if not plot_type: + plot_type = 'png' + + # check if existing plots need to be redone + redo_plot = adfobj.get_basic_info('redo_plot') + print(f"\t NOTE: redo_plot is set to {redo_plot}") + + #Grab saved files + obs_ds = xr.open_dataset('/glade/derecho/scratch/mdfowler/ENSOmetrics_Obs.nc') + cesm1_ds = xr.open_dataset("/glade/derecho/scratch/mdfowler/ENSOmetrics_CESM1.nc") + cesm2_ds = xr.open_dataset("/glade/derecho/scratch/mdfowler/ENSOmetrics_CESM2.nc") + dev_ds = xr.open_dataset("/glade/derecho/scratch/mdfowler/ENSOmetrics_CESM3dev.nc") + + # + + + + + + + + + + + + + + + + + + + + + # Make Nino variance comparison plots + # + + + + + + + + + + + + + + + + + + + + +# J F M A M J J A S O N D + daysPerMonth = np.asarray([31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]) + + ninoSelString = ['Nino34', 'Nino12', 'Nino3', 'Nino4'] + + for iNino in range(len(ninoSelString)): + #Set path for variance figures: + plot_loc_ts = Path(plot_locations[0]) / f'NinoVarianceComparison_{ninoSelString[iNino]}_ENSO_Mean.{plot_type}' + print('Creating plot for ', ninoSelString[iNino]) + + # Check redo_plot. If set to True: remove old plots, if they already exist: + if (not redo_plot) and plot_loc_ts.is_file(): + #Add already-existing plot to website (if enabled): + adfobj.debug_log(f"'{plot_loc_ts}' exists and clobber is false.") + adfobj.add_website_data(plot_loc_ts, "NinoVarianceComparison", None, season=ninoSelString[iNino], multi_case=True, non_season=True, plot_type = "ENSO") + + #Continue to next iteration: + return + elif (redo_plot): + if plot_loc_ts.is_file(): + plot_loc_ts.unlink() + #End if + + if ( (ninoSelString[iNino]=='Nino34') ): + pltVar = 'nino34_variance' + elif ( (ninoSelString[iNino]=='Nino12') ): + pltVar = 'nino12_variance' + elif ( (ninoSelString[iNino]=='Nino3') ): + pltVar = 'nino3_variance' + elif ( (ninoSelString[iNino]=='Nino4') ): + pltVar = 'nino4_variance' + else: + print('Invalid choice for ninoSelString supplied.\n Valid options: Nino34, Nino12, Nino3, Nino4') + + fig,axs=plt.subplots(1,1,figsize=(15,5)) + + for iCase in range(len(dev_ds.case.values)): + case_max = np.nanmax(dev_ds[pltVar].values[iCase,:]) + case_min = np.nanmin(dev_ds[pltVar].values[iCase,:]) + # Weighted mean + weights = ( daysPerMonth / daysPerMonth.sum() ) + weighted_mean = (dev_ds[pltVar].values[iCase,:] * weights).sum() / weights.sum() + + axs.plot(iCase+np.ones(2), [case_min, case_max], 'k-') + axs.plot(iCase+1, weighted_mean,'o', color='k') + axs.plot(iCase+1, case_min,'^',color='k') + axs.plot(iCase+1, case_max,'v',color='k') + + cesm1_xCenter = len(dev_ds.case.values)+1.5 + cesm2_xCenter = len(dev_ds.case.values)+3 + obs_xCenter = len(dev_ds.case.values)+4 + + offset = np.linspace(cesm1_xCenter-0.5, cesm1_xCenter+0.5, len(cesm1_ds.event.values)) + for iEvent in range(len(cesm1_ds.event.values)): + case_max = np.nanmax(cesm1_ds[pltVar].values[iEvent,:]) + case_min = np.nanmin(cesm1_ds[pltVar].values[iEvent,:]) + # Weighted mean + weights = ( daysPerMonth / daysPerMonth.sum() ) + weighted_mean = (cesm1_ds[pltVar].values[iEvent,:] * weights).sum() / weights.sum() + + axs.plot(offset[iEvent]*np.ones(2), [case_min, case_max], + '-', color='mediumpurple', alpha=0.2) + axs.plot(offset[iEvent], weighted_mean,'o', color='mediumpurple',alpha=0.4) + axs.plot(offset[iEvent], case_min,'^',color='mediumpurple',alpha=0.4) + axs.plot(offset[iEvent],case_max,'v',color='mediumpurple',alpha=0.4) + + + offset = np.linspace(cesm2_xCenter-0.5, cesm2_xCenter+0.5, len(cesm2_ds.event.values)) + for iEvent in range(len(cesm2_ds.event.values)): + case_max = np.nanmax(cesm2_ds[pltVar].values[iEvent,:]) + case_min = np.nanmin(cesm2_ds[pltVar].values[iEvent,:]) + # Weighted mean + weights = ( daysPerMonth / daysPerMonth.sum() ) + weighted_mean = (cesm2_ds[pltVar].values[iEvent,:] * weights).sum() / weights.sum() + + axs.plot(offset[iEvent]*np.ones(2), [case_min, case_max], + '-', color='orange', alpha=0.2) + axs.plot(offset[iEvent], weighted_mean,'o', color='orange',alpha=0.4) + axs.plot(offset[iEvent], case_min,'^',color='orange',alpha=0.4) + axs.plot(offset[iEvent],case_max,'v',color='orange',alpha=0.4) + + + # Get obs + obs_max = np.nanmax(obs_ds[pltVar].values) + obs_min = np.nanmin(obs_ds[pltVar].values) + # Weighted mean + weights = ( daysPerMonth / daysPerMonth.sum() ) + weighted_mean = (obs_ds[pltVar].values * weights).sum() / weights.sum() + + axs.plot(obs_xCenter*np.ones(2), [obs_min, obs_max], '-', color='firebrick') + axs.plot(obs_xCenter, weighted_mean,'o', color='firebrick') + axs.plot(obs_xCenter, obs_min,'^',color='firebrick') + axs.plot(obs_xCenter, obs_max,'v',color='firebrick') + + ## General plot settings + ticks = np.append(1+np.arange(len(dev_ds.case.values)), cesm1_xCenter) + ticks = np.append(ticks, cesm2_xCenter) + ticks = np.append(ticks, obs_xCenter) + + tickLabels = np.append(dev_ds.case.values, 'CESM1') + tickLabels = np.append(tickLabels, 'CESM2') + tickLabels = np.append(tickLabels, 'HadiSST') + + axs.set_xticks(ticks) + axs.set_xticklabels(tickLabels) + plt.setp( axs.xaxis.get_majorticklabels(), rotation=45 ) + + axs.axhline(obs_min, color='firebrick',alpha=0.3) + axs.axhline(obs_max, color='firebrick',alpha=0.3) + + axs.set_title('Monthly '+ninoSelString[iNino]+' variance') + + #Save figure to file: + fig.savefig(plot_loc_ts, bbox_inches='tight', facecolor='white') + + #Add plot to website (if enabled): + adfobj.add_website_data(plot_loc_ts, "NinoVarianceComparison", None, season=ninoSelString[iNino], multi_case=True, non_season=True, plot_type = "ENSO") + + # + + + + + + + + + + + + + + + + + + + + + # Make autocorrelation comparison plot + # + + + + + + + + + + + + + + + + + + + + + + #Set path for variance figures: + plot_loc_autocorr = Path(plot_locations[0]) / f'NinoAutocorrelation_Nino34_ENSO_Mean.{plot_type}' + print('Creating plot for Nino 3.4 autocorrelation transition') + + # Check redo_plot. If set to True: remove old plots, if they already exist: + if (not redo_plot) and plot_loc_autocorr.is_file(): + #Add already-existing plot to website (if enabled): + adfobj.debug_log(f"'{plot_loc_autocorr}' exists and clobber is false.") + adfobj.add_website_data(plot_loc_autocorr, "NinoAutocorrelation", None, season='Nino34', multi_case=True, non_season=True, plot_type = "ENSO") + + #Continue to next iteration: + return + elif (redo_plot): + if plot_loc_autocorr.is_file(): + plot_loc_autocorr.unlink() + #End if + + fig,axs=plt.subplots(1,1,figsize=(15,5)) + + axs.scatter(dev_ds.case.values, dev_ds.nino34_transMonth, color='k') + axs.plot(np.full([len(cesm1_ds.event.values)], 'CESM1'), cesm1_ds.nino34_transMonth, 'o', color='mediumpurple', alpha=0.4) + axs.plot(np.full([len(cesm2_ds.event.values)], 'CESM2'), cesm2_ds.nino34_transMonth, 'o', color='orange', alpha=0.4) + axs.scatter('HadiSST', obs_ds.nino34_transMonth, color='k', marker='*', s=200) + axs.axhline(obs_ds.nino34_transMonth, color='grey',alpha=0.5) + axs.set_title('Month of transition in Autocorrelation for Nino3.4', fontsize=14) + axs.set_ylabel('Lag of nMonths',fontsize=12) + + ticks = np.arange(len(dev_ds.case.values)+3) + + tickLabels = np.append(dev_ds.case.values, 'CESM1') + tickLabels = np.append(tickLabels, 'CESM2') + tickLabels = np.append(tickLabels, 'HadiSST') + + axs.set_xticks(ticks) + axs.set_xticklabels(tickLabels, fontsize=11) + axs.set_ylim([5,30]) + plt.setp( axs.xaxis.get_majorticklabels(), rotation=40 ) + + #Save figure to file: + fig.savefig(plot_loc_autocorr, bbox_inches='tight', facecolor='white') + + #Add plot to website (if enabled): + adfobj.add_website_data(plot_loc_autocorr, "NinoAutocorrelation", None, season='Nino34', multi_case=True, non_season=True, plot_type = "ENSO") + + + # + + + + + + + + + + + + + + + + + + + + + # Make western extent comparison plot + # + + + + + + + + + + + + + + + + + + + + + + #Set path for variance figures: + plot_loc_westext = Path(plot_locations[0]) / f'WesternExtent_Nino34_ENSO_Mean.{plot_type}' + print('Creating plot for Nino3.4 SST anomaly western extent') + + # Check redo_plot. If set to True: remove old plots, if they already exist: + if (not redo_plot) and plot_loc_westext.is_file(): + #Add already-existing plot to website (if enabled): + adfobj.debug_log(f"'{plot_loc_westext}' exists and clobber is false.") + adfobj.add_website_data(plot_loc_westext, "WesternExtent", None, season='Nino34', multi_case=True, non_season=True, plot_type = "ENSO") + + #Continue to next iteration: + return + elif (redo_plot): + if plot_loc_westext.is_file(): + plot_loc_westext.unlink() + #End if + + fig,axs=plt.subplots(1,2,figsize=(10,9)) + axs = axs.ravel() + + axs[0].scatter(dev_ds.nino34_zeroContour, dev_ds.case.values, color='dodgerblue', alpha=1) + axs[0].set_title('Westernmost longitude of \n0 contour related to Nino 3.4 max') + axs[0].invert_yaxis() + axs[0].plot(cesm1_ds.nino34_zeroContour, np.full([len(cesm1_ds.event.values)], 'CESM1'), 'o', color='mediumpurple', alpha=0.4) + axs[0].plot(cesm2_ds.nino34_zeroContour, np.full([len(cesm2_ds.event.values)], 'CESM2'), 'o', color='orange', alpha=0.4) + axs[0].plot(obs_ds.nino34_zeroContour, 'HadiSST', '*', color='k',markersize=14) + + axs[1].scatter(dev_ds.nino34_0p5Contour, dev_ds.case.values, marker = 's', color='dodgerblue', alpha=1) + axs[1].set_title('Westernmost longitude of \n0.5 contour related to Nino 3.4 max') + axs[1].invert_yaxis() + axs[1].plot(cesm1_ds.nino34_0p5Contour, np.full([len(cesm1_ds.event.values)], 'CESM1'), 's', color='mediumpurple', alpha=0.4) + axs[1].plot(cesm2_ds.nino34_0p5Contour, np.full([len(cesm2_ds.event.values)], 'CESM2'), 's', color='orange', alpha=0.4) + axs[1].plot(obs_ds.nino34_0p5Contour, 'HadiSST', '*', color='k',markersize=14) + + axs[0].set_xlim(120,170) + axs[1].set_xlim(120,170) + + fig.subplots_adjust(wspace=0.5) + + #Save figure to file: + fig.savefig(plot_loc_westext, bbox_inches='tight', facecolor='white') + + #Add plot to website (if enabled): + adfobj.add_website_data(plot_loc_westext, "WesternExtent", None, season='Nino34', multi_case=True, non_season=True, plot_type = "ENSO") + + # + + + + + + + + + + + + + + + + + + + + + # Make scatterplot comparison plots + # + + + + + + + + + + + + + + + + + + + + + case_shortName = adfobj.get_cam_info("case_nickname", required=True)[0] + + #Set path for variance figures: + plot_loc_coldPoolScat = Path(plot_locations[0]) / f'Scatter_ColdPoolExtent_ENSO_Mean.{plot_type}' + print('Creating plot for cold pool extent vs. western extent') + + # Check redo_plot. If set to True: remove old plots, if they already exist: + if (not redo_plot) and plot_loc_coldPoolScat.is_file(): + #Add already-existing plot to website (if enabled): + adfobj.debug_log(f"'{plot_loc_coldPoolScat}' exists and clobber is false.") + adfobj.add_website_data(plot_loc_coldPoolScat, "Scatter", None, season='ColdPoolExtent', multi_case=True, non_season=True, plot_type = "ENSO") + + #Continue to next iteration: + return + elif (redo_plot): + if plot_loc_coldPoolScat.is_file(): + plot_loc_coldPoolScat.unlink() + #End if + + fig,axs = plt.subplots(1,1, figsize=(10,8)) + + xArr = np.append(dev_ds.sst_lon299, obs_ds.sst_lon299) + yArr = np.append(dev_ds.nino34_0p5Contour, obs_ds.nino34_0p5Contour) + cArr = np.append(dev_ds.sst_raw_nino34.mean(dim='season').values, (obs_ds.sst_raw_nino34.mean(dim='season').values + 273.15) ) + + caseLabels = np.append(dev_ds.case.values, 'HadiSST') + axs.tick_params(labelsize=12) # Adjust 12 to your desired tick font size + s = axs.scatter(xArr, yArr, c=cArr) + cb = fig.colorbar(s,ax=axs,orientation='vertical') + cb.set_label('Mean Nino3.4 SST', size=14) + cb.ax.tick_params(labelsize=12) # Adjust 12 to your desired tick font size + + for iCase in range(len(xArr)): + if caseLabels[iCase]=='HadiSST': + axs.text(xArr[iCase]*0.975, + yArr[iCase], + # corrs.case.values[iCase], color='k', alpha=0.5, fontsize=11) + caseLabels[iCase], color='g', fontsize=11) + elif caseLabels[iCase]==case_shortName: + axs.text(xArr[iCase], + yArr[iCase], + # corrs.case.values[iCase], color='k', alpha=0.5, fontsize=11) + caseLabels[iCase], color='r', alpha=1, fontsize=14) + else: + axs.text(xArr[iCase], + yArr[iCase], + # corrs.case.values[iCase], color='k', alpha=0.5, fontsize=11) + caseLabels[iCase], color='k', alpha=0.7, fontsize=12) + + + axs.set_title('Extent of cold tongue vs. Western extent of 0.5 contour for ENSO',fontsize=14) + axs.set_xlabel('Westernmost point of 299 K contour in SSTs',fontsize=12) + axs.set_ylabel('Westernmost point of 0.5 contour\n in SST anomaly correlations with nino3.4 index', fontsize=12) + + #Save figure to file: + fig.savefig(plot_loc_coldPoolScat, bbox_inches='tight', facecolor='white') + + #Add plot to website (if enabled): + adfobj.add_website_data(plot_loc_coldPoolScat, "Scatter", None, season='ColdPoolExtent', multi_case=True, non_season=True, plot_type = "ENSO") + + + # --- Next Scatter: Autocorrelation and mean SST ---- + plot_loc_durationSST = Path(plot_locations[0]) / f'Scatter_NinoDuration&meanSST_ENSO_Mean.{plot_type}' + print('Creating plot for Nino duration and mean SST') + + # Check redo_plot. If set to True: remove old plots, if they already exist: + if (not redo_plot) and plot_loc_durationSST.is_file(): + #Add already-existing plot to website (if enabled): + adfobj.debug_log(f"'{plot_loc_durationSST}' exists and clobber is false.") + adfobj.add_website_data(plot_loc_durationSST, "Scatter", None, season='NinoDuration&meanSST', multi_case=True, non_season=True, plot_type = "ENSO") + + #Continue to next iteration: + return + elif (redo_plot): + if plot_loc_durationSST.is_file(): + plot_loc_durationSST.unlink() + #End if + + fig,axs = plt.subplots(1,1,figsize=(8,5)) + + selSeason = 'DJF' + + sst_raw_season = dev_ds.sst_raw.sel(season=selSeason).copy(deep=True) + transit_month = dev_ds.nino34_transMonth + # sst_cesm2_raw_season = sst_cesm2.sel(season=selSeason) + + # Define bounding box + sst_sel = sst_raw_season.sel(lat=slice(-5,5), lon=slice(120,280)) + sst_sel_mean = sst_sel.mean(dim='lon').mean(dim='lat') + axs.set_title('Averages over '+selSeason+' & Full Pacific (-5S to 5N, 120 to 280)') + + axs.scatter(sst_sel_mean.values, transit_month.values) + + for iCase in range(len(sst_raw_season.case.values)): + if caseLabels[iCase]==case_shortName: + axs.plot(sst_sel_mean.values[iCase], + transit_month[iCase], 'ro') + axs.text(sst_sel_mean.values[iCase], + transit_month[iCase], + dev_ds.case.values[iCase], color='r', alpha=1, fontsize=14) + else: + if transit_month[iCase]<=25: + axs.text(sst_sel_mean.values[iCase], + transit_month[iCase], + dev_ds.case.values[iCase], color='k', alpha=0.4, fontsize=12) + + axs.set_xlabel('Mean SST') + axs.set_ylabel('Autocorrelation Transition (nMonths)') + axs.set_ylim([7,25]) + + axs.plot(273.15+obs_ds.sst_obs_raw_seasonal.sel(season=selSeason,lat=slice(-5,5), lon=slice(120,280)).mean(dim='lon').mean(dim='lat').values, + obs_ds.nino34_transMonth.values, 'g*', markersize=12) + axs.text(273.15+obs_ds.sst_obs_raw_seasonal.sel(season=selSeason,lat=slice(-5,5), lon=slice(120,280)).mean(dim='lon').mean(dim='lat').values, + obs_ds.nino34_transMonth.values*0.91, + 'HadSST', color='g', alpha=0.8, fontsize=12) + + ## Add CESM ensembles + axs.scatter(cesm2_ds.sst_raw.sel(season=selSeason,lat=slice(-5,5), lon=slice(120,280)).mean(dim='lon').mean(dim='lat').values, + cesm2_ds.nino34_transMonth.values, marker='s', c='orange', alpha=0.8) + axs.text(np.nanmean(cesm2_ds.sst_raw.sel(season=selSeason,lat=slice(-5,5), lon=slice(120,280)).mean(dim='lon').mean(dim='lat').values)*1.001, + np.nanmean(cesm2_ds.nino34_transMonth.values), 'CESM2', color='orange', alpha=0.8, fontsize=12) + + + axs.scatter(cesm1_ds.sst_raw.sel(season=selSeason,lat=slice(-5,5), lon=slice(120,280)).mean(dim='lon').mean(dim='lat').values, + cesm1_ds.nino34_transMonth.values, marker='^', c='mediumorchid', alpha=0.8) + axs.text(np.nanmean(cesm1_ds.sst_raw.sel(season=selSeason,lat=slice(-5,5), lon=slice(120,280)).mean(dim='lon').mean(dim='lat').values)*0.998, + np.nanmean(cesm1_ds.nino34_transMonth.values), 'CESM1', color='mediumorchid', alpha=0.8, fontsize=12) + + #Save figure to file: + fig.savefig(plot_loc_durationSST, bbox_inches='tight', facecolor='white') + + #Add plot to website (if enabled): + adfobj.add_website_data(plot_loc_durationSST, "Scatter", None, season='NinoDuration&meanSST', multi_case=True, non_season=True, plot_type = "ENSO") + + return \ No newline at end of file From 0d632c8fa4eb494fdc00f55fa981c78fdd6e73da Mon Sep 17 00:00:00 2001 From: justin-richling <56696811+justin-richling@users.noreply.github.com> Date: Fri, 25 Apr 2025 17:06:39 -0600 Subject: [PATCH 29/91] Update version of ADF conda env With the change in xesmf change version number --- env/conda_environment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/env/conda_environment.yaml b/env/conda_environment.yaml index 1ff0c70a4..9245929fe 100644 --- a/env/conda_environment.yaml +++ b/env/conda_environment.yaml @@ -1,4 +1,4 @@ -name: adf_v0.11 +name: adf_v0.12 channels: - conda-forge - defaults @@ -14,5 +14,5 @@ dependencies: - xskillscore=0.0.5 - geocat-comp=2022.08.0 - python=3.11 - - xesm=0.8.7 -prefix: /glade/work/$USER/conda-envs/adf_v0.11 + - xesmf=0.8.7 +prefix: /glade/work/$USER/conda-envs/adf_v0.12 From 0459bf1315bd8a650fb85b4d7a0e78f35bc8b81b Mon Sep 17 00:00:00 2001 From: justin-richling <56696811+justin-richling@users.noreply.github.com> Date: Fri, 25 Apr 2025 17:07:15 -0600 Subject: [PATCH 30/91] Update README.md for newest env version number --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4060749ad..0213afc06 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ If you are using conda on a non-CISL machine, then you can create and activate t ``` conda env create -f env/conda_environment.yaml -conda activate adf_v0.11 +conda activate adf_v0.12 ``` Also, along with these python requirements, the `ncrcat` NetCDF Operator (NCO) is also needed. On the CISL machines this can be loaded by simply running: From 371cfebe000805d2e8927b8572443c0a97dbed0f Mon Sep 17 00:00:00 2001 From: Tomas Torsvik Date: Thu, 1 May 2025 15:01:35 +0200 Subject: [PATCH 31/91] Update conda environment for geocat-comp 2024.04.0 --- env/conda_environment.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/env/conda_environment.yaml b/env/conda_environment.yaml index 9245929fe..af18c197b 100644 --- a/env/conda_environment.yaml +++ b/env/conda_environment.yaml @@ -1,18 +1,18 @@ -name: adf_v0.12 +name: adf_v0.13 channels: - conda-forge - defaults dependencies: - - pyyaml=6.0 - - scipy=1.10.0 - - cartopy=0.21.1 - - netcdf4=1.6.2 - - xarray=2023.1.0 - - matplotlib=3.6.3 - - pandas=1.5.3 - - pint=0.16 #GeoCAT doesn't work with newer versions - - xskillscore=0.0.5 - - geocat-comp=2022.08.0 - - python=3.11 - - xesmf=0.8.7 -prefix: /glade/work/$USER/conda-envs/adf_v0.12 + - pyyaml=6.0.2 + - scipy=1.12.0 + - cartopy=0.23.0 + - netcdf4=1.6.5 + - xarray=2024.1.1 + - matplotlib=3.9.4 + - pandas=2.2.0 + - pint=0.23 + - xskillscore=0.0.24 + - geocat-comp=2024.04.0 + - python=3.12 + - xesmf>=0.8.8 +prefix: /glade/work/$USER/conda-envs/adf_v0.13 From ee70ddb96dc306800be5fe6fa3bf4d3f9904a0d3 Mon Sep 17 00:00:00 2001 From: Tomas Torsvik Date: Thu, 1 May 2025 19:15:35 +0200 Subject: [PATCH 32/91] Add support for uxarray=2025.03.0 --- env/conda_environment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/env/conda_environment.yaml b/env/conda_environment.yaml index af18c197b..f232e1fb8 100644 --- a/env/conda_environment.yaml +++ b/env/conda_environment.yaml @@ -8,6 +8,7 @@ dependencies: - cartopy=0.23.0 - netcdf4=1.6.5 - xarray=2024.1.1 + - uxarray=2025.03.0 - matplotlib=3.9.4 - pandas=2.2.0 - pint=0.23 From ca9d2a8f8740a4f3c40f9623ed7f439232470b5a Mon Sep 17 00:00:00 2001 From: justin-richling <56696811+justin-richling@users.noreply.github.com> Date: Tue, 3 Jun 2025 09:36:35 -0600 Subject: [PATCH 33/91] Update ADF conda env version number --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0213afc06..3a5b6e85a 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ If you are using conda on a non-CISL machine, then you can create and activate t ``` conda env create -f env/conda_environment.yaml -conda activate adf_v0.12 +conda activate adf_v0.13 ``` Also, along with these python requirements, the `ncrcat` NetCDF Operator (NCO) is also needed. On the CISL machines this can be loaded by simply running: From 7b617dd5d2b4d0d70fbf2732e7652fcda098f6d9 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 5 Jun 2025 14:54:00 -0600 Subject: [PATCH 34/91] Docstrings added to create_climo_files.py --- scripts/averaging/create_climo_files.py | 105 +++++++++++++++++++----- 1 file changed, 83 insertions(+), 22 deletions(-) diff --git a/scripts/averaging/create_climo_files.py b/scripts/averaging/create_climo_files.py index f9f05454c..e8420dcda 100644 --- a/scripts/averaging/create_climo_files.py +++ b/scripts/averaging/create_climo_files.py @@ -8,12 +8,39 @@ def my_formatwarning(msg, *args, **kwargs): return str(msg) + '\n' warnings.formatwarning = my_formatwarning +from uuid import int_ +from mypyc.ir.rtypes import optional_value_type +from operator import truediv import numpy as np import xarray as xr # module-level import so all functions can get to it. import multiprocessing as mp def get_time_slice_by_year(time, startyear, endyear): + """ + Return slice object inclusive of specified start and end years. + + Parameters + ---------- + time + Time coordinate variable, expects xarray `dt` accessor. + startyear : int + The year to start the slice + endyear : int + The year to end the slice + + Returns + ------- + slice + slice object with start and end years specified; + if `dt` accessor is available values will be time indices + + Notes + ----- + When the `dt` accessor is not available, instead of indices + returns `slice(startyear, endyear)` and prints a warning + since this is unlikely to actually work. + """ if not hasattr(time, 'dt'): print("Warning: get_time_slice_by_year requires the `time` parameter to be an xarray time coordinate with a dt accessor. Returning generic slice (which will probably fail).") return slice(startyear, endyear) @@ -29,29 +56,37 @@ def get_time_slice_by_year(time, startyear, endyear): def create_climo_files(adf, clobber=False, search=None): """ - This is an example function showing - how to set-up a time-averaging method - for calculating climatologies from - CAM time series files using - multiprocessing for parallelization. + Orchestrates production of monthly climatology files + from CAM time series files. + + Parameters + ---------- + adf + the ADF object + clobber : bool, optional + Overwrite existing climatology files if truediv. + Defaults to False (do not delete). + search : str, optional + optional string used as a template to find the time series files + using {CASE} and {VARIABLE} and otherwise an arbitrary shell-like globbing pattern: + example 1: provide the string "{CASE}.*.{VARIABLE}.*.nc" this is the default + example 2: maybe CASE is not necessary because post-process destroyed the info "post_process_text-{VARIABLE}.nc" + example 3: order does not matter "{VARIABLE}.{CASE}.*.nc" + Only CASE and VARIABLE are allowed because they are arguments to the averaging function + + Notes + ----- + No return value; produces netCDF files. + + Uses multiprocessing for parallelization. + + Calls local function `process_variable` for calculation. Description of needed inputs from ADF: - - case_name -> Name of CAM case provided by "cam_case_name" - input_ts_loc -> Location of CAM time series files provided by "cam_ts_loc" - output_loc -> Location to write CAM climo files to, provided by "cam_climo_loc" - var_list -> List of CAM output variables provided by "diag_var_list" - - Optional keyword arguments: - - clobber -> whether to overwrite existing climatology files. Defaults to False (do not delete). - - search -> optional; if supplied requires a string used as a template to find the time series files - using {CASE} and {VARIABLE} and otherwise an arbitrary shell-like globbing pattern: - example 1: provide the string "{CASE}.*.{VARIABLE}.*.nc" this is the default - example 2: maybe CASE is not necessary because post-process destroyed the info "post_process_text-{VARIABLE}.nc" - example 3: order does not matter "{VARIABLE}.{CASE}.*.nc" - Only CASE and VARIABLE are allowed because they are arguments to the averaging function + case_name -> Name of CAM case provided by "cam_case_name" + input_ts_loc -> Location of CAM time series files provided by "cam_ts_loc" + output_loc -> Location to write CAM climo files to, provided by "cam_climo_loc" + var_list -> List of CAM output variables provided by "diag_var_list" """ #Import necessary modules: @@ -215,7 +250,20 @@ def create_climo_files(adf, clobber=False, search=None): # def process_variable(adf, ts_files, syr, eyr, output_file): ''' - Compute and save the climatology file. + Compute and save the monthly climatology file. + + Parameters + ---------- + adf + The ADF object + ts_files : list + list of paths to time series files + syr : str + start year, with leading zeros if needed + eyr : str + end year, with leading zeros if needed + output_file : str or Path + file path for output climatology file ''' #Read in files via xarray (xr): if len(ts_files) == 1: @@ -259,6 +307,19 @@ def process_variable(adf, ts_files, syr, eyr, output_file): def check_averaging_interval(syear_in, eyear_in): + """ + Parameters + ---------- + syear_in + start year, should be convertible to int + eyear_in + end year, should be convertible to int + + Returns + ------- + tuple + (start_year, end_year) as str with leading zeros included if needed (4-digit) + """ #For now, make sure year inputs are integers or None, #in order to allow for the zero additions done below: if syear_in: From 5fb38a82fa7b54521f51d5d3383082aa1246fbdf Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 25 Jun 2025 15:53:53 -0600 Subject: [PATCH 35/91] Use xr.DataArray.data as arguments to np functions Some combinations of recent versions of xarray and numpy throw a ValueError when the first two arguments to np.linspace are DataArray objects instead of numpy.array objects. This can be avoided by using the .data component as the argument. I've verified that in older numpy versions where the xarray argument is allowed, the result from np.linspace is bit-for-bit identical whether the argument is the DataArray or DataArray.data --- lib/plotting_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/plotting_functions.py b/lib/plotting_functions.py index 08de58860..b59f54599 100644 --- a/lib/plotting_functions.py +++ b/lib/plotting_functions.py @@ -1902,10 +1902,10 @@ def prep_contour_plot(adata, bdata, diffdata, pctdata, **kwargs): levelsdiff = np.arange(*kwargs['diff_contour_range']) else: # set a symmetric color bar for diff: - absmaxdif = np.max(np.abs(diffdata)) + absmaxdif = np.max(np.abs(diffdata.data)) # set levels for difference plot: levelsdiff = np.linspace(-1*absmaxdif, absmaxdif, 12) - + # Percent Difference options -- Check in kwargs for colormap and levels if "pct_diff_colormap" in kwargs: cmappct = kwargs["pct_diff_colormap"] From a51851f03da38c5f7c4b32484095fba30ff720c2 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 26 Jun 2025 14:08:28 -0600 Subject: [PATCH 36/91] clean up unneeded imports --- scripts/averaging/create_climo_files.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/scripts/averaging/create_climo_files.py b/scripts/averaging/create_climo_files.py index e8420dcda..11dfc5dc0 100644 --- a/scripts/averaging/create_climo_files.py +++ b/scripts/averaging/create_climo_files.py @@ -1,21 +1,16 @@ -################## -#Warnings function -################## - +""" +Module to create (monthly) climatology files. +""" import warnings # use to warn user about missing files. +import multiprocessing as mp +import numpy as np +import xarray as xr # module-level import so all functions can get to it. + def my_formatwarning(msg, *args, **kwargs): # ignore everything except the message return str(msg) + '\n' warnings.formatwarning = my_formatwarning -from uuid import int_ -from mypyc.ir.rtypes import optional_value_type -from operator import truediv -import numpy as np -import xarray as xr # module-level import so all functions can get to it. - -import multiprocessing as mp - def get_time_slice_by_year(time, startyear, endyear): """ Return slice object inclusive of specified start and end years. @@ -64,7 +59,7 @@ def create_climo_files(adf, clobber=False, search=None): adf the ADF object clobber : bool, optional - Overwrite existing climatology files if truediv. + Overwrite existing climatology files if true. Defaults to False (do not delete). search : str, optional optional string used as a template to find the time series files From 77cb0d97a40f6bc83630c5b0758c553e033da12e Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 26 Jun 2025 15:07:12 -0600 Subject: [PATCH 37/91] updated docstrings for regrid_and_vert_interp.py --- scripts/regridding/regrid_and_vert_interp.py | 125 ++++++++++++------- 1 file changed, 81 insertions(+), 44 deletions(-) diff --git a/scripts/regridding/regrid_and_vert_interp.py b/scripts/regridding/regrid_and_vert_interp.py index 8316cb3f1..c1a6aa665 100644 --- a/scripts/regridding/regrid_and_vert_interp.py +++ b/scripts/regridding/regrid_and_vert_interp.py @@ -1,33 +1,30 @@ -#Import standard modules: +"""Driver for horizontal and vertical interpolation. +""" import xarray as xr def regrid_and_vert_interp(adf): """ - This funtion regrids the test cases to the same horizontal - grid as the observations or baseline climatology. It then - vertically interpolates the test case (and baseline case - if need be) to match a default set of pressure levels, which - are (in hPa): + Regrids the test cases to the same horizontal + grid as the reference climatology and vertically + interpolates the test case (and reference if needed) + to match a default set of pressure levels (in hPa). + Parameters + ---------- + adf + The ADF object + + + Notes + ----- + Default pressure levels: 1000, 925, 850, 700, 500, 400, 300, 250, 200, 150, 100, 70, 50, 30, 20, 10, 7, 5, 3, 2, 1 Currently any 3-D observations file needs to have equivalent pressure levels in order to work properly, although in the future it is hoped to enable the vertical interpolation of observations as well. - - Description of needed inputs from ADF: - - case_name -> Name of CAM case provided by "cam_case_name" - input_climo_loc -> Location of CAM climo files provided by "cam_climo_loc" - output_loc -> Location to write re-gridded CAM files, specified by "cam_regrid_loc" - var_list -> List of CAM output variables provided by "diag_var_list" - var_defaults -> Dict that has keys that are variable names and values that are plotting preferences/defaults. - target_list -> List of target data sets CAM could be regridded to - taget_loc -> Location of target files that CAM will be regridded to - overwrite_regrid -> Logical to determine if already existing re-gridded - files will be overwritten. Specified by "cam_overwrite_regrid" """ #Import necessary modules: @@ -396,30 +393,35 @@ def regrid_and_vert_interp(adf): #Helper functions ################# -def _regrid_and_interpolate_levs(model_dataset, var_name, regrid_dataset=None, regrid_ofrac=False, **kwargs): +def _regrid_and_interpolate_levs(model_dataset, var_name, regrid_dataset=None, **kwargs): """ Function that takes a variable from a model xarray dataset, regrids it to another dataset's lat/lon coordinates (if applicable), and then interpolates it vertically to a set of pre-defined pressure levels. - ---------- - model_dataset -> The xarray dataset which contains the model variable data - var_name -> The name of the variable to be regridded/interpolated. - - Optional inputs: - - ps_file -> A NetCDF file containing already re-gridded surface pressure - regrid_dataset -> The xarray dataset that contains the lat/lon grid that - "var_name" will be regridded to. If not present then - only the vertical interpolation will be done. - - kwargs -> Keyword arguments that contain paths to surface pressure - and mid-level pressure files, which are necessary for - certain types of vertical interpolation. - This function returns a new xarray dataset that contains the regridded - and/or vertically-interpolated model variable. + Parameters + ---------- + model_dataset : xarray.Dataset + The xarray dataset which contains the model variable data + var_name : str + The name of the variable to be regridded/interpolated. + regrid_dataset : xr.Dataset or xr.DataArray, optional + The xarray object that contains the destination lat/lon grid + If not present then only vertical interpolation will be performed. + **kwargs + Additional optional arguments: + - `ps_file` : str or Path + specify surface pressure netCDF file + - `pmid_file` : str or Path + specify vertical layer midpoint pressure netCDF file + + Returns + ------- + xarray.Dataset + This function returns a new xarray dataset that contains the regridded + and/or vertically-interpolated model variable. """ #Import ADF-specific functions: @@ -654,12 +656,23 @@ def _regrid_and_interpolate_levs(model_dataset, var_name, regrid_dataset=None, r #Return dataset: return rgdata_interp -##### def save_to_nc(tosave, outname, attrs=None, proc=None): - """Saves xarray variable to new netCDF file""" + """Saves xarray variable to new netCDF file + + Parameters + ---------- + tosave : xarray.Dataset or xarray.DataArray + data to write to file + outname : str or Path + output netCDF file path + attrs : dict, optional + attributes dictionary for data + proc : str, optional + string to append to "Processing_info" attribute + """ - xo = tosave # used to have more stuff here. + xo = tosave # deal with getting non-nan fill values. if isinstance(xo, xr.Dataset): enc_dv = {xname: {'_FillValue': None} for xname in xo.data_vars} @@ -674,10 +687,37 @@ def save_to_nc(tosave, outname, attrs=None, proc=None): xo.attrs['Processing_info'] = f"Start from file {origname}. " + proc xo.to_netcdf(outname, format='NETCDF4', encoding=enc) -##### def regrid_data(fromthis, tothis, method=1): - """Regrid data using various different methods""" + """Regrid between lat-lon grids using various different methods + + Parameters + ---------- + fromthis : xarray.DataArray + original data + tothis : xarray.DataArray + provides destination grid information (regular lat-lon) + method : int, optional + method to use for regridding + 1 - xarray, `interp_like` + 2 - xarray, `interp` + 3 - xESMF, `Regridder()` + 4 - GeoCAT, `linint2` (may be deprecated) + + Returns + ------- + xarray.DataArray + Data interpolated to destination grid + + Notes + ----- + 1. xarray's interpolation does not respect longitude's periodicity + 2. xESMF can sometimes malfunction depending on dependencies + 3. GeoCAT `linint2` might be deprecated + + A more robust regridding solution is being explored. + + """ if method == 1: # kludgy: spatial regridding only, seems like can't automatically deal with time @@ -706,7 +746,4 @@ def regrid_data(fromthis, tothis, method=1): newlon = tothis['lon'] result = geocat.comp.linint2(fromthis, newlon, newlat, False) result.name = fromthis.name - return result - #End if - -##### \ No newline at end of file + return result \ No newline at end of file From 02b62d3b033c6959c43ee9dd6a343603c88abf32 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 27 Jun 2025 13:44:08 -0600 Subject: [PATCH 38/91] Correct polar map issue with transform_first. Makes faster! --- lib/plotting_functions.py | 74 +++++++++++++++++++++++++++-------- scripts/plotting/polar_map.py | 41 +++++++++++++++---- 2 files changed, 91 insertions(+), 24 deletions(-) diff --git a/lib/plotting_functions.py b/lib/plotting_functions.py index 08de58860..df46706fc 100644 --- a/lib/plotting_functions.py +++ b/lib/plotting_functions.py @@ -304,6 +304,38 @@ def get_central_longitude(*args): return 180 #End if + +def transform_coordinates_for_projection(proj, lon, lat): + """ + Explicitly project coordinates using the projection object. + + Parameters + ---------- + proj : cartopy.ccrs.CRS + projection object + lat : xarray.DataArray or numpy.ndarray + latitudes (in degrees) + lon :array.DataArray or numpy.ndarray + longitudes (in degrees) + + Returns + ------- + x_proj : numpy.ndarray + array of projected longitudes + y_proj : numpy.ndarray + array of projected latitudes + + Notes + ----- + This is what cartopy's transform_first=True *should* be doing internally. + We find that it sometimes fails for polar plots, so do it with this manually. + This dramatically speeds up polar plots. + """ + lons, lats = np.meshgrid(lon, lat) + x_proj, y_proj, _ = proj.transform_points(ccrs.PlateCarree(), lons, lats).T # .T to unpack, .T again to get x,y,z arrays + return x_proj.T, y_proj.T + + ####### def global_average(fld, wgt, verbose=False): @@ -603,10 +635,18 @@ def domain_stats(data, domain): x_region_max = x_region.max().item() return x_region_mean, x_region_max, x_region_min -def make_polar_plot(wks, case_nickname, base_nickname, - case_climo_yrs, baseline_climo_yrs, - d1:xr.DataArray, d2:xr.DataArray, difference:Optional[xr.DataArray]=None,pctchange:Optional[xr.DataArray]=None, - domain:Optional[list]=None, hemisphere:Optional[str]=None, obs=False, **kwargs): +def make_polar_plot(wks, case_nickname, + base_nickname, + case_climo_yrs, + baseline_climo_yrs, + d1:xr.DataArray, + d2:xr.DataArray, + difference:Optional[xr.DataArray]=None, + pctchange:Optional[xr.DataArray]=None, + domain:Optional[list]=None, + hemisphere:Optional[str]=None, + obs=False, + **kwargs): """Make a stereographic polar plot for the given data and hemisphere. @@ -727,7 +767,7 @@ def make_polar_plot(wks, case_nickname, base_nickname, levelsdiff = np.arange(*kwargs['diff_contour_range']) else: # set levels for difference plot (with a symmetric color bar): - levelsdiff = np.linspace(-1*absmaxdif, absmaxdif, 12) + levelsdiff = np.linspace(-1*absmaxdif.data, absmaxdif.data, 12) #End if if "pct_diff_contour_levels" in kwargs: @@ -758,7 +798,7 @@ def make_polar_plot(wks, case_nickname, base_nickname, #End if if max(np.abs(levelsdiff)) > 10*absmaxdif: - levelsdiff = np.linspace(-1*absmaxdif, absmaxdif, 12) + levelsdiff = np.linspace(-1*absmaxdif.data, absmaxdif.data, 12) #End if @@ -779,8 +819,7 @@ def make_polar_plot(wks, case_nickname, base_nickname, #End if # -- end options - - lons, lats = np.meshgrid(lon_cyclic, d1.lat) + lons, lats = transform_coordinates_for_projection(proj, lon_cyclic, d1.lat) # Explicit coordinate transform fig = plt.figure(figsize=(10,10)) gs = mpl.gridspec.GridSpec(2, 4, wspace=0.9) @@ -794,27 +833,28 @@ def make_polar_plot(wks, case_nickname, base_nickname, levs_diff = np.unique(np.array(levelsdiff)) levs_pctdiff = np.unique(np.array(levelspctdiff)) + # BPM: removing `transform=ccrs.PlateCarree()` from contourf calls & transform_first=True if len(levs) < 2: - img1 = ax1.contourf(lons, lats, d1_cyclic, transform=ccrs.PlateCarree(), colors="w", norm=norm1) + img1 = ax1.contourf(lons, lats, d1_cyclic, colors="w", norm=norm1) ax1.text(0.4, 0.4, empty_message, transform=ax1.transAxes, bbox=props) - img2 = ax2.contourf(lons, lats, d2_cyclic, transform=ccrs.PlateCarree(), colors="w", norm=norm1) + img2 = ax2.contourf(lons, lats, d2_cyclic, colors="w", norm=norm1) ax2.text(0.4, 0.4, empty_message, transform=ax2.transAxes, bbox=props) else: - img1 = ax1.contourf(lons, lats, d1_cyclic, transform=ccrs.PlateCarree(), cmap=cmap1, norm=norm1, levels=levels1) - img2 = ax2.contourf(lons, lats, d2_cyclic, transform=ccrs.PlateCarree(), cmap=cmap1, norm=norm1, levels=levels1) + img1 = ax1.contourf(lons, lats, d1_cyclic, cmap=cmap1, norm=norm1, levels=levels1) + img2 = ax2.contourf(lons, lats, d2_cyclic, cmap=cmap1, norm=norm1, levels=levels1) if len(levs_pctdiff) < 2: - img3 = ax3.contourf(lons, lats, pct_cyclic, transform=ccrs.PlateCarree(), colors="w", norm=pctnorm, transform_first=True) + img3 = ax3.contourf(lons, lats, pct_cyclic, colors="w", norm=pctnorm) ax3.text(0.4, 0.4, empty_message, transform=ax3.transAxes, bbox=props) else: - img3 = ax3.contourf(lons, lats, pct_cyclic, transform=ccrs.PlateCarree(), cmap=cmappct, norm=pctnorm, levels=levelspctdiff, transform_first=True) + img3 = ax3.contourf(lons, lats, pct_cyclic, cmap=cmappct, norm=pctnorm, levels=levelspctdiff) if len(levs_diff) < 2: - img4 = ax4.contourf(lons, lats, dif_cyclic, transform=ccrs.PlateCarree(), colors="w", norm=dnorm) + img4 = ax4.contourf(lons, lats, dif_cyclic, colors="w", norm=dnorm) ax4.text(0.4, 0.4, empty_message, transform=ax4.transAxes, bbox=props) else: - img4 = ax4.contourf(lons, lats, dif_cyclic, transform=ccrs.PlateCarree(), cmap=cmapdiff, norm=dnorm, levels=levelsdiff) + img4 = ax4.contourf(lons, lats, dif_cyclic, cmap=cmapdiff, norm=dnorm, levels=levelsdiff) #Set Main title for subplots: st = fig.suptitle(wks.stem[:-5].replace("_"," - "), fontsize=18) @@ -1904,7 +1944,7 @@ def prep_contour_plot(adata, bdata, diffdata, pctdata, **kwargs): # set a symmetric color bar for diff: absmaxdif = np.max(np.abs(diffdata)) # set levels for difference plot: - levelsdiff = np.linspace(-1*absmaxdif, absmaxdif, 12) + levelsdiff = np.linspace(-1*absmaxdif.data, absmaxdif.data, 12) # Percent Difference options -- Check in kwargs for colormap and levels if "pct_diff_colormap" in kwargs: diff --git a/scripts/plotting/polar_map.py b/scripts/plotting/polar_map.py index c99e5ac9a..932f7fc92 100644 --- a/scripts/plotting/polar_map.py +++ b/scripts/plotting/polar_map.py @@ -1,6 +1,5 @@ -from pathlib import Path # python standard library - -# data loading / analysis +"""Module to make polar stereographic maps.""" +from pathlib import Path import xarray as xr import numpy as np @@ -8,11 +7,39 @@ import plotting_functions as pf def get_hemisphere(hemi_type): - """Helper function to convert plot type to hemisphere code.""" + """Helper function to convert plot type to hemisphere code. + + Parameters + ---------- + hemi_type : str + if `NHPolar` set NH, otherwise SH + + Returns + ------- + str + NH or SH + """ return "NH" if hemi_type == "NHPolar" else "SH" -def process_seasonal_data(mdata, odata, season, vres): - """Helper function to calculate seasonal means and differences.""" +def process_seasonal_data(mdata, odata, season): + """Helper function to calculate seasonal means and differences. + Parameters + ---------- + mdata : xarray.DataArray + test case data + odata : xarray.DataArray + reference case data + season : str + season (JJA, DJF, MAM, SON) + + Returns + ------- + mseason : xarray.DataArray + oseason : xarray.DataArray + dseason : xarray.DataArray + pseason : xarray.DataArray + Seasonal means for test, reference, difference, and percent difference + """ mseason = pf.seasonal_mean(mdata, season=season, is_climo=True) oseason = pf.seasonal_mean(odata, season=season, is_climo=True) @@ -212,7 +239,7 @@ def polar_map(adfobj): mseason, oseason, dseason, pseason = process_seasonal_data( mdata, use_odata, - plot['season'], vres + plot['season'] ) # Create plot From a97a188724d916f76cb388f82d2930675e2db514 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 27 Jun 2025 16:10:32 -0600 Subject: [PATCH 39/91] log_p fix, as in PR #380 --- lib/plotting_functions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/plotting_functions.py b/lib/plotting_functions.py index df46706fc..4c30534d1 100644 --- a/lib/plotting_functions.py +++ b/lib/plotting_functions.py @@ -2007,7 +2007,7 @@ def prep_contour_plot(adata, bdata, diffdata, pctdata, **kwargs): def plot_zonal_mean_and_save(wks, case_nickname, base_nickname, case_climo_yrs, baseline_climo_yrs, - adata, bdata, has_lev, log_p, obs=False, **kwargs): + adata, bdata, has_lev, log_p=False, obs=False, **kwargs): """This is the default zonal mean plot @@ -2185,7 +2185,7 @@ def plot_zonal_mean_and_save(wks, case_nickname, base_nickname, def plot_meridional_mean_and_save(wks, case_nickname, base_nickname, case_climo_yrs, baseline_climo_yrs, - adata, bdata, has_lev, latbounds=None, obs=False, **kwargs): + adata, bdata, has_lev, log_p=False, latbounds=None, obs=False, **kwargs): """Default meridional mean plot @@ -2209,6 +2209,8 @@ def plot_meridional_mean_and_save(wks, case_nickname, base_nickname, It must have the same dimensions and vertical levels as adata. has_lev : bool whether lev dimension is present + log_p : bool, optional + (Not implemented) use log(pressure) vertical axis latbounds : numbers.Number or slice, optional indicates latitude bounds to average over if it is a number, assume symmetric about equator, From 3dec31f3d7f2a2a72ca221ec7c0ca5eed6b68217 Mon Sep 17 00:00:00 2001 From: Behrooz-Roozitalab Date: Sat, 5 Jul 2025 10:00:01 -0600 Subject: [PATCH 40/91] Major fixes to the chemistry table budget --- scripts/analysis/aerosol_gas_tables.py | 225 ++++++++++++++----------- 1 file changed, 131 insertions(+), 94 deletions(-) diff --git a/scripts/analysis/aerosol_gas_tables.py b/scripts/analysis/aerosol_gas_tables.py index 44f8afc51..32719bb1d 100644 --- a/scripts/analysis/aerosol_gas_tables.py +++ b/scripts/analysis/aerosol_gas_tables.py @@ -94,6 +94,15 @@ def aerosol_gas_tables(adfobj): * lifetime inconsitencies * Removed redundant calculations to improve the speed * Verified the results against the NCL script. + + Behrooz Roozitalab, 5 Jun, 2025 + - fixed: + * Fix the bugs in the calculation (when converting from Jupyterhub to ADF) + * add the 'U' variable in dic_SE + * make the code faster by modifying make_Dic_scn_var_comp + * Add a condition to calculate whole world budgets when O3 is not find. + * Update pressure calculation in a more general way. + ''' @@ -106,7 +115,6 @@ def aerosol_gas_tables(adfobj): # Variable defaults info res = adfobj.variable_defaults # dict of variable-specific plot preferences bres = res['budget_tables'] - # list of the gaseous variables to be caculated. GAS_VARIABLES = bres['GAS_VARIABLES'] @@ -158,7 +166,6 @@ def aerosol_gas_tables(adfobj): # Grab all case nickname(s) test_nicknames_list = adfobj.case_nicknames["test_nicknames"] nicknames_list = test_nicknames_list - # Grab climo years start_years = adfobj.climo_yrs["syears"] end_years = adfobj.climo_yrs["eyears"] @@ -168,7 +175,7 @@ def aerosol_gas_tables(adfobj): # Grab history file locations from config yaml file hist_locs = adfobj.get_cam_info("cam_hist_loc", required=True) - + # Check if this is test model vs baseline model # If so, update test case(s) lists created above if not adfobj.compare_obs: @@ -184,7 +191,7 @@ def aerosol_gas_tables(adfobj): hist_strs += [adfobj.hist_string["base_hist_str"]] hist_locs += [adfobj.get_baseline_info("cam_hist_loc")] # End if - + # Check to ensure number of case names matches number history file locations. # If not, exit script if len(hist_locs) != len(case_names): @@ -192,13 +199,12 @@ def aerosol_gas_tables(adfobj): raise AdfError(errmsg) # Initialize nicknames dictionary - nicknames = {} + #nicknames = {} # Filter the list to include only strings that are possible h0 strings # - Search for either h0 or h0a substrings = {"cam.h0","cam.h0a"} case_hist_strs = [] - print("hist_strs",hist_strs,"\n") for cam_case_str in hist_strs: # Check each possible h0 string for string in cam_case_str: @@ -208,9 +214,10 @@ def aerosol_gas_tables(adfobj): # Create path object for the CAM history file(s) location: data_dirs = [] - for case_idx,case in enumerate(case_names): + for case_idx,case in enumerate(nicknames_list): + print(f"\t Looking for history location: {hist_locs[case_idx]}") - nicknames[case] = nicknames_list[case_idx] + #Check that history file input directory actually exists: if (hist_locs[case_idx] is None) or (not Path(hist_locs[case_idx]).is_dir()): @@ -237,9 +244,10 @@ def aerosol_gas_tables(adfobj): areas = {} trops = {} insides = {} - for i,case in enumerate(case_names): + for i,case in enumerate(nicknames_list): + start_year = start_years[i] - end_year = end_years[i] + end_year = end_years[i] + 1 start_date = f"{start_year}-1-1" end_date = f"{end_year}-1-1" @@ -248,17 +256,17 @@ def aerosol_gas_tables(adfobj): end_period = datetime.strptime(end_date, "%Y-%m-%d") # Calculated duration of time period in seconds? - durations[case_names[i]] = (end_period-start_period).days*86400+365*86400 + durations[case] = (end_period-start_period).days*86400 #+365*86400 + # Get number of years for calculations - num_yrs[case_names[i]] = (int(end_year)-int(start_year))+1 + num_yrs[case] = (int(end_year)-int(start_year)) #+1 # Get currenty history file directory data_dir = data_dirs[i] # Get all files, lats, lons, and area weights for current case Files,Lats,Lons,areas[case],ext1_SE = Get_files(adfobj,data_dir,start_year,end_year,case_hist_strs[i],area=True) - # find the name of all the variables in the file. # this will help the code to work for the variables that are not in the files (assingn 0s) tmp_file = xr.open_dataset(Path(data_dir) / Files[0]) @@ -274,8 +282,7 @@ def aerosol_gas_tables(adfobj): # Gather dictionary data for current case # NOTE: The calculations can take a long time... - Dic_crit, Dic_scn_var_comp[case] = make_Dic_scn_var_comp(adfobj, VARIABLES, data_dir, dic_SE, Files, ext1_SE, AEROSOLS) - + Dic_crit, Dic_scn_var_comp[case],Tropospheric = make_Dic_scn_var_comp(adfobj, VARIABLES, data_dir, dic_SE, Files, ext1_SE, AEROSOLS,Tropospheric) # Regional refinement # NOTE: This function 'Inside_SE' is unavailable at the moment! - JR 10/2024 if regional: @@ -288,8 +295,7 @@ def aerosol_gas_tables(adfobj): inside = np.full((len(Lats),len(Lons)),True) # Set critical threshold - current_crit = Dic_crit[0] - + current_crit = Dic_crit if Tropospheric: trop = np.where(current_crit>150,np.nan,current_crit) #strat=np.where(current_crit>150,current_crit,np.nan) @@ -304,12 +310,14 @@ def aerosol_gas_tables(adfobj): "areas":areas, "trops":trops, "case_names":case_names, - "nicknames":nicknames, + "nicknames":nicknames_list, "durations":durations, "insides":insides, "num_yrs":num_yrs, "AEROSOLS":AEROSOLS} + #print(table_kwargs) + # Create the budget tables #------------------------- # Aerosols @@ -334,7 +342,7 @@ def list_files(adfobj, directory, start_year ,end_year, h_case): """ # History file year range - yrs = np.arange(int(start_year), int(end_year)+1) + yrs = np.arange(int(start_year), int(end_year)) all_filenames = [] for i in yrs: @@ -364,7 +372,6 @@ def Get_files(adfobj, data_dir, start_year, end_year, h_case, **kwargs): Earth_rad=6.371e6 # Earth Radius in meters current_files = list_files(adfobj, data_dir, start_year, end_year,h_case) - # get the Lat and Lons for each case tmp_file = xr.open_dataset(Path(data_dir) / current_files[0]) lon = tmp_file['lon'+ext1_SE].data @@ -380,8 +387,9 @@ def Get_files(adfobj, data_dir, start_year, end_year, h_case, **kwargs): except KeyError: try: tmp_area = tmp_file['AREA'+ext1_SE].isel(time=0).data - Earth_area = 4 * np.pi * Earth_rad**(2) - areas = tmp_area*Earth_area/np.nansum(tmp_area) + areas=tmp_area + #Earth_area = 4 * np.pi * Earth_rad**(2) + #areas = tmp_area*Earth_area/np.nansum(tmp_area) except: dlon = np.abs(lon[1]-lon[0]) dlat = np.abs(lat[1]-lat[0]) @@ -411,6 +419,7 @@ def set_dic_SE(ListVars, ext1_SE): # Chemistry #---------- + dic_SE['U']={'U'+ext1_SE:1} dic_SE['O3']={'O3'+ext1_SE:1e9} # covert to ppb for Tropopause calculation dic_SE['CH4']={'CH4'+ext1_SE:1} dic_SE['CO']={'CO'+ext1_SE:1} @@ -420,7 +429,8 @@ def set_dic_SE(ListVars, ext1_SE): dic_SE['CH3OH']={'CH3OH'+ext1_SE:1} dic_SE['CH3COCH3']={'CH3COCH3'+ext1_SE:1} dic_SE['CH3CCL3']={'CH3CCL3'+ext1_SE:1} - + dic_SE['CHBR3']={'CHBR3'+ext1_SE:1} + dic_SE['CH2BR2']={'CH2BR2'+ext1_SE:1} # Aerosols #--------- @@ -777,7 +787,7 @@ def fill_dic_SE(adfobj, dic_SE, variables, ListVars, ext1_SE, AEROSOLS, MW, AVO, ##### -def make_Dic_scn_var_comp(adfobj, variables, current_dir, dic_SE, current_files, ext1_SE, AEROSOLS): +def make_Dic_scn_var_comp(adfobj, variables, current_dir, dic_SE, current_files, ext1_SE, AEROSOLS,Tropospheric): """ This function retrieves the files, latitude, and longitude information in all the directories within the chosen dates. @@ -861,14 +871,13 @@ def make_Dic_scn_var_comp(adfobj, variables, current_dir, dic_SE, current_files, msg += f"\n\t Derived components for CAM variable {current_var}: {components}" #adfobj.debug_log(msg) Dic_comp={} + Dic_comp,missing_vars,needed_vars=SEbudget(adfobj,dic_SE,current_dir,current_files,components,ext1_SE) + for comp in components: # Write details to log file msg += f"\n\t\t calculate derived component: {comp} for main variable, {current_var}" adfobj.debug_log(msg) - # Get component values - current_data,missing_vars,needed_vars = SEbudget(adfobj,dic_SE,current_dir,current_files,comp,ext1_SE) - # Gather info for debugging for var_m in missing_vars: if var_m not in missing_vars_tot: @@ -878,9 +887,6 @@ def make_Dic_scn_var_comp(adfobj, variables, current_dir, dic_SE, current_files, needed_vars_tot.append(var_n) # End for # End for - #TODO: check this section to see if it can't be better run - # Set dictionary for component - Dic_comp[comp] = current_data # Set dictionary for key of current variable with dictionary values of all # necessary constituents for calculating the current variable Dic_var_comp[current_var] = Dic_comp @@ -888,9 +894,16 @@ def make_Dic_scn_var_comp(adfobj, variables, current_dir, dic_SE, current_files, # Critical threshholds, just run this once # this is for finding tropospheric values - current_crit=SEbudget(adfobj,dic_SE,current_dir,current_files,'O3',ext1_SE) - Dic_crit=current_crit - + # Critical threshholds?\n", + # Just run this once\n", + try: + current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['O3'],ext1_SE) + Dic_crit=current_crit['O3'] + except: + current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['U'],ext1_SE) + Dic_crit=current_crit['U'] + Tropospheric=False + msg += f"\n\t WARNING: O3 was not found in the model, budgets are total column" # Log info to logging file msg = f"chem/aerosol tables:" msg += f"\n\t - potential missing variables from budget? {missing_vars_tot}" @@ -900,17 +913,16 @@ def make_Dic_scn_var_comp(adfobj, variables, current_dir, dic_SE, current_files, msg += f"\n\t - needed variables for budget {needed_vars_tot}" adfobj.debug_log(msg) - return Dic_crit,Dic_scn_var_comp + return Dic_crit,Dic_scn_var_comp,Tropospheric ##### -def SEbudget(adfobj,dic_SE,data_dir,files,var,ext1_SE,**kwargs): +def SEbudget(adfobj,dic_SE,data_dir,files,vars,ext1_SE,**kwargs): """ Function used for getting the data for the budget calculation. This is the chunk of code that takes the longest by far. Example: - ~70/75 mins per case for 9 years ** This is for both chemistry and aeorosl calculations dic_SE: dictionary specyfing what variables to get. For example, @@ -937,8 +949,9 @@ def SEbudget(adfobj,dic_SE,data_dir,files,var,ext1_SE,**kwargs): # Set lists to gather necessary variables for logging missing_vars = [] needed_vars = [] + Dic_all_data={} - all_data=[] +# all_data=[] for file in range(len(files)): ds=xr.open_dataset(Path(data_dir) / files[file]) @@ -948,33 +961,6 @@ def SEbudget(adfobj,dic_SE,data_dir,files,var,ext1_SE,**kwargs): mock_2d=np.zeros_like(np.array(ds['PS'+ext1_SE].isel(time=0))) mock_3d=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) - # Star gathering of variable data - data=[] - for i in dic_SE[var].keys(): - if i not in needed_vars: - needed_vars.append(i) - if file == 0: - msg = f"chem/aerosol tables: 'SEbudget'" - msg += f"\n\t\t ** variable(s) needed for derived var {var}: {dic_SE[var].keys()}" - msg += f"\n\t\t - constituent for derived var {var}: {i}" - adfobj.debug_log(msg) - - if ((i!='PS'+ext1_SE) and (i!='U'+ext1_SE) ) : - data.append(np.array(ds[i].isel(time=0))*dic_SE[var][i]) - else: - if i=='PS'+ext1_SE: - data.append(mock_2d) - else: - data.append(mock_3d) - # End if - - if var not in missing_vars: - missing_vars.append(var) - # End if - - # Get total summed data for this history file data - data=np.sum(data,axis=0) - try: delP=np.array(ds['PDELDRY'+ext1_SE].isel(time=0)) except: @@ -989,32 +975,81 @@ def SEbudget(adfobj,dic_SE,data_dir,files,var,ext1_SE,**kwargs): # End try/except P0=1e5 - Plevel=np.zeros_like(np.array(ds['U'+ext1_SE])) + Plevel=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) for i in range(len(Plevel)): Plevel[i]=hyai[i]*P0+hybi[i]*PS delP=Plevel[1:]-Plevel[:-1] + + for var in vars: + if file == 0: + Dic_all_data[var]=[] + + + # Star gathering of variable data + data=[] + for i in dic_SE[var].keys(): + + if file == 0: + msg = f"chem/aerosol tables: 'SEbudget'" + msg += f"\n\t\t ** variable(s) needed for derived var {var}: {dic_SE[var].keys()}" + msg += f"\n\t\t - constituent for derived var {var}: {i}" + adfobj.debug_log(msg) + if i not in needed_vars: + needed_vars.append(i) + if ((i!='PS'+ext1_SE) and (i!='U'+ext1_SE) ) : + data.append(np.array(ds[i].isel(time=0))*dic_SE[var][i]) + else: + if i=='PS'+ext1_SE: + data.append(mock_2d) + else: + data.append(mock_3d) + # End if + if file == 0: + + if var not in missing_vars: + if var!='U': # This is to avoid confusion between U variable or U mock! + missing_vars.append(var) + msg += f"\n\t\t - no variable was found for var {var}: {i}" + + # End if + + # Get total summed data for this history file data + data=np.sum(data,axis=0) + # End try/except - if ('CHML' in var) or ('CHMP' in var) : - Temp=np.array(ds['T'+ext1_SE].isel(time=0)) - Pres=np.array(ds['PMID'+ext1_SE].isel(time=0)) - rho= Pres/(Rgas*Temp) - data=data*delP/rho - elif ('BURDEN' in var): - data=data*delP - else: - data=data + if ('CHML' in var) or ('CHMP' in var) : + Temp=np.array(ds['T'+ext1_SE].isel(time=0)) + try: + Pres=np.array(ds['PMID'+ext1_SE].isel(time=0)) + except: + hyam=np.array(ds['hyam']) + hybm=np.array(ds['hybm']) + try: + PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) + except: + PS=np.array(ds['PS'+ext1_SE].isel(time=0)) + P0=1e5 + Pres=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) + for i in range(len(Pres)): + Pres[i]=hyam[i]*P0+hybm[i]*PS + rho= Pres/(Rgas*Temp) + data=data*delP/rho + elif ('BURDEN' in var): + data=data*delP + else: + data=data # End if + # Add data to list + Dic_all_data[var].append(data) + ds.close() + for var in vars: # Take mean + Dic_all_data[var]=np.nanmean(Dic_all_data[var],axis=0) - # Add data to list - all_data.append(data) - # Take mean - all_data=np.nanmean(all_data,axis=0) - - return all_data,missing_vars,needed_vars + return Dic_all_data,missing_vars,needed_vars ##### @@ -1113,8 +1148,8 @@ def calc_budget_data(current_var, Dic_scn_var_comp, area, trop, inside, num_yrs, GAEX = np.ma.sum(gaex*duration*1e-9)/num_yrs chem_dict[f"{current_var}_GAEX (Tg{specifier}/yr)"] = np.round(GAEX,5) - # LifeTime = Burden/(loss+deposition) - LT = BURDEN/(CHML+DDF-WDF)* duration/86400/num_yrs # days + # LifeTime = Burden/(loss+deposition) no chemical loss for aerosols + LT = BURDEN/(DDF+WDF)* duration/86400/num_yrs # days chem_dict[f"{current_var}_LIFETIME (days)"] = np.round(LT,5) if current_var == 'SO4': @@ -1150,17 +1185,17 @@ def calc_budget_data(current_var, Dic_scn_var_comp, area, trop, inside, num_yrs, chem_dict[f"{current_var}_WETDEP (Tg/yr)"] = np.round(WDF,5) # Total Deposition - TDEP = DDF - WDF + TDEP = DDF + WDF chem_dict[f"{current_var}_TDEP (Tg/yr)"] = np.round(TDEP,5) # LifeTime = Burden/(loss+deposition) if current_var == "CH4": - LT = BURDEN/(CHML+DDF-WDF) # years + LT = BURDEN/(CHML+DDF+WDF) # years chem_dict[f"{current_var}_LIFETIME (years)"] = np.round(LT,5) else: - if (CHML+DDF-WDF) > 0: + if (CHML+DDF+WDF) > 0: if CHML != 0: - LT = BURDEN/(CHML+DDF-WDF)*duration/86400/num_yrs # days + LT = BURDEN/(CHML+DDF+WDF)*duration/86400/num_yrs # days chem_dict[f"{current_var}_LIFETIME (days)"] = np.round(LT,5) else: # do not report lifetime if chemical loss (for gases) is not included in the model outputs @@ -1221,16 +1256,17 @@ def make_table(adfobj, vars, chem_type, Dic_scn_var_comp, areas, trops, case_nam output_location = Path(output_locs[0]) # Loop over model cases - for case in case_names: - nickname = nicknames[case] + + for i,case in enumerate(nicknames): + + nickname = case # Collect row data in a list of dictionaries - durations[case] + #durations[case] rows = [] for current_var in vars: chem_dict = calc_budget_data(current_var, Dic_scn_var_comp[case], areas[case], trops[case], insides[case], num_yrs[case], durations[case], AEROSOLS) - # Loop through table variables for key, val in chem_dict.items(): if val != 0: # Skip variables with a value of 0 @@ -1260,19 +1296,20 @@ def make_table(adfobj, vars, chem_type, Dic_scn_var_comp, areas, trops, case_nam # Store the DataFrame in the dictionary dfs[nickname] = table_df + # End for # Merge the DataFrames on the 'variable' column if len(case_names) == 2: - table_df = pd.merge(dfs[nicknames[case_names[0]]], dfs[nicknames[case_names[1]]], on='variable') + + table_df = pd.merge(dfs[nicknames[0]], dfs[nicknames[1]], on='variable') # Calculate the differences between case columns - table_df['difference'] = table_df[nicknames[case_names[0]]] - table_df[nicknames[case_names[1]]] + table_df['difference'] = table_df[nicknames[0]] - table_df[nicknames[1]] #Create output file name: output_csv_file = output_location / f'ADF_amwg_{chem_type}_table.csv' - # Save table to CSV and add table dataframe to website (if enabled) table_df.to_csv(output_csv_file, index=False) adfobj.add_website_data(table_df, chem_type, case, plot_type="Tables") -##### \ No newline at end of file +##### From af2225526bd1fccc0ca000edcf99b6cc5bacd6cd Mon Sep 17 00:00:00 2001 From: justin-richling <56696811+justin-richling@users.noreply.github.com> Date: Wed, 23 Jul 2025 09:52:38 -0600 Subject: [PATCH 41/91] Remove print statement --- scripts/plotting/polar_map.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/plotting/polar_map.py b/scripts/plotting/polar_map.py index 932f7fc92..fd7d3de1d 100644 --- a/scripts/plotting/polar_map.py +++ b/scripts/plotting/polar_map.py @@ -154,7 +154,6 @@ def polar_map(adfobj): for s in seasons: for hemi_type in ["NHPolar", "SHPolar"]: if pres_levs and has_lev: # 3-D variable & pressure levels specified - print(f"POLAR: {pres_levs = }") for pres in pres_levs: plot_name = plot_loc / f"{var}_{pres}hpa_{s}_{hemi_type}_Mean.{plot_type}" info = { @@ -268,4 +267,4 @@ def polar_map(adfobj): #END OF `polar_map` function ############## -# END OF FILE \ No newline at end of file +# END OF FILE From faa9b17d895c2197635421fb235f7a244861f524 Mon Sep 17 00:00:00 2001 From: justin-richling <56696811+justin-richling@users.noreply.github.com> Date: Wed, 23 Jul 2025 11:26:55 -0600 Subject: [PATCH 42/91] Update global_latlon_map.py Currently this is checking incorrectly for lat/lon dims if the variable is 3d --- scripts/plotting/global_latlon_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/plotting/global_latlon_map.py b/scripts/plotting/global_latlon_map.py index 8f1f3b06b..dc78ecec2 100644 --- a/scripts/plotting/global_latlon_map.py +++ b/scripts/plotting/global_latlon_map.py @@ -158,7 +158,7 @@ def process_case(adfobj, case_name, case_idx, var, odata, seasons, return has_dims = pf.validate_dims(mdata, ["lat", "lon", "lev"]) - if not pf.lat_lon_validate_dims(mdata): + if (not has_dims['has_lat']) or (not has_dims['has_lon']): print(f"\t WARNING: Model data missing lat/lon dimensions") return From eb32086fcf5e1edff43c7015f6588ef3982a5a64 Mon Sep 17 00:00:00 2001 From: justin-richling <56696811+justin-richling@users.noreply.github.com> Date: Wed, 23 Jul 2025 11:38:16 -0600 Subject: [PATCH 43/91] Fix incorrect colorbar axis This will fix the warning given during AOD lat/lon plots --- scripts/plotting/aod_latlon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/plotting/aod_latlon.py b/scripts/plotting/aod_latlon.py index c0c293c1f..462242037 100644 --- a/scripts/plotting/aod_latlon.py +++ b/scripts/plotting/aod_latlon.py @@ -435,7 +435,8 @@ def aod_panel_latlon(adfobj, plot_titles, plot_params, data, season, obs_name, c ax.coastlines() ax.set_title(title, fontsize=10) - cbar = plt.colorbar(img, orientation='horizontal', pad=0.05) + fig_to_use = ind_fig if not is_panel else fig + cbar = fig_to_use.colorbar(img, ax=ax, orientation='horizontal', pad=0.05) if 'ticks' in plot_param: cbar.set_ticks(plot_param['ticks']) if 'tick_labels' in plot_param: From 6d1289e02c21ce50723070015500b0f990ddd08e Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Fri, 1 Aug 2025 17:25:30 -0600 Subject: [PATCH 44/91] Keep changes in adf_variable_defaults.yaml --- lib/adf_variable_defaults.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index 60743359c..3ddd188c4 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -206,6 +206,9 @@ BURDENSO4: category: "Aerosols" pct_diff_contour_levels: [-100,-75,-50,-40,-30,-20,-10,-8,-6,-4,-2,0,2,4,6,8,10,20,30,40,50,75,100] pct_diff_colormap: "PuOr_r" + obs_file: "BURDENSO4_CAMS_monthly_climo_1degree_200301-202412.nc" + obs_name: "CAMS" + obs_var_name: "BURDENSO4" BURDENSOA: category: "Aerosols" From 5b361e76d40d924dc3c7c26e866eedb4a27311f7 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Mon, 4 Aug 2025 12:02:08 -0600 Subject: [PATCH 45/91] Added CAMS dust burden as observational reference --- lib/adf_variable_defaults.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index 466f68855..eeb58d0b3 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -190,8 +190,19 @@ BURDENBC: BURDENDUST: category: "Aerosols" + colormap: "plasma_r" + scale_factor: 1000000 + add_offset: 0 + new_unit: 'g m$^{-2}$' pct_diff_contour_levels: [-100,-75,-50,-40,-30,-20,-10,-8,-6,-4,-2,0,2,4,6,8,10,20,30,40,50,75,100] pct_diff_colormap: "PuOr_r" + diff_contour_range: [-1000, 1100, 100] + diff_colormap: "RdBu_r" + obs_file: "BURDENDUST_CAMS_monthly_climo_1degree_200301-202412.nc" + obs_var_name: "BURDENDUST" + obs_name: "CAMS" + obs_scale_factor: 1000000 + obs_add_offset: 0 BURDENPOM: category: "Aerosols" From da0ff483f52b6b3fbce79474787deb1f37e4184e Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Mon, 4 Aug 2025 12:31:41 -0600 Subject: [PATCH 46/91] Added CAMS SEASALT aerosol burden to variable defaults --- lib/adf_variable_defaults.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index eeb58d0b3..7c815f293 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -211,8 +211,19 @@ BURDENPOM: BURDENSEASALT: category: "Aerosols" + colormap: "plasma_r" + scale_factor: 1000000 + add_offset: 0 + new_unit: 'g m$^{-2}$' + diff_contour_range: [-200, 200, 25] + diff_colormap: "RdBu_r" pct_diff_contour_levels: [-100,-75,-50,-40,-30,-20,-10,-8,-6,-4,-2,0,2,4,6,8,10,20,30,40,50,75,100] pct_diff_colormap: "PuOr_r" + obs_file: "BURDENSEASALT_CAMS_monthly_climo_1degree_200301-202412.nc" + obs_var_name: "BURDENSEASALT" + obs_name: "CAMS" + obs_scale_factor: 1000000 + obs_add_offset: 0 BURDENSO4: category: "Aerosols" From ac03ed4d6f0ff456e9d1e79580b134814c0c1784 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 7 Aug 2025 15:01:39 -0600 Subject: [PATCH 47/91] include BURDENBC (MERRA-2); update units for BURDENs --- lib/adf_variable_defaults.yaml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index 7c815f293..9795f6fc2 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -185,8 +185,18 @@ AODVISdn: BURDENBC: category: "Aerosols" + colormap: "plasma_r" + scale_factor: 1000000 + add_offset: 0 + new_unit: 'g m$^{-2}$' pct_diff_contour_levels: [-100,-75,-50,-40,-30,-20,-10,-8,-6,-4,-2,0,2,4,6,8,10,20,30,40,50,75,100] pct_diff_colormap: "PuOr_r" + diff_colormap: "RdBu_r" + obs_file: "BURDENBC_MERRA2_monthly_climo_1degree_200001-202506.nc" + obs_var_name: "BURDENBC" + obs_name: "MERRA2" + obs_scale_factor: 1000000 + obs_add_offset: 0 BURDENDUST: category: "Aerosols" @@ -206,6 +216,10 @@ BURDENDUST: BURDENPOM: category: "Aerosols" + colormap: "plasma_r" + scale_factor: 1000000 + add_offset: 0 + new_unit: 'g m$^{-2}$' pct_diff_contour_levels: [-100,-75,-50,-40,-30,-20,-10,-8,-6,-4,-2,0,2,4,6,8,10,20,30,40,50,75,100] pct_diff_colormap: "PuOr_r" @@ -227,16 +241,28 @@ BURDENSEASALT: BURDENSO4: category: "Aerosols" + colormap: "plasma_r" + scale_factor: 1000000 + add_offset: 0 + new_unit: 'g m$^{-2}$' pct_diff_contour_levels: [-100,-75,-50,-40,-30,-20,-10,-8,-6,-4,-2,0,2,4,6,8,10,20,30,40,50,75,100] pct_diff_colormap: "PuOr_r" + diff_colormap: "RdBu_r" obs_file: "BURDENSO4_CAMS_monthly_climo_1degree_200301-202412.nc" obs_name: "CAMS" obs_var_name: "BURDENSO4" + obs_scale_factor: 1000000 + obs_add_offset: 0 BURDENSOA: category: "Aerosols" + colormap: "plasma_r" + scale_factor: 1000000 + add_offset: 0 + new_unit: 'g m$^{-2}$' pct_diff_contour_levels: [-100,-75,-50,-40,-30,-20,-10,-8,-6,-4,-2,0,2,4,6,8,10,20,30,40,50,75,100] pct_diff_colormap: "PuOr_r" + diff_colormap: "RdBu_r" DMS: category: "Aerosols" From 74efbc89e917b4fa6dbcd21d00c9536407914b48 Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 7 Aug 2025 15:06:12 -0600 Subject: [PATCH 48/91] diff colors for BURDENPOM --- lib/adf_variable_defaults.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index 9795f6fc2..7222e1ae4 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -222,6 +222,8 @@ BURDENPOM: new_unit: 'g m$^{-2}$' pct_diff_contour_levels: [-100,-75,-50,-40,-30,-20,-10,-8,-6,-4,-2,0,2,4,6,8,10,20,30,40,50,75,100] pct_diff_colormap: "PuOr_r" + diff_colormap: "RdBu_r" + BURDENSEASALT: category: "Aerosols" From aae12ccec32c64ecedafb19d9da5fa805d718bda Mon Sep 17 00:00:00 2001 From: Brian Medeiros Date: Thu, 7 Aug 2025 16:06:59 -0600 Subject: [PATCH 49/91] Add CALIPSO cloud fields to adf_variable_defaults.yaml --- lib/adf_variable_defaults.yaml | 65 ++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index 7222e1ae4..6dcf5800b 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -2322,6 +2322,71 @@ TOT_ICLD_VISTAU: pct_diff_contour_levels: [-100,-75,-50,-40,-30,-20,-10,-8,-6,-4,-2,0,2,4,6,8,10,20,30,40,50,75,100] pct_diff_colormap: "PuOr_r" +CLDTOT_CAL: + colormap: "cividis" + contour_levels_range: [0, 105, 5] + diff_colormap: "RdBu_r" + diff_contour_range: [-40, 40, 5] + scale_factor: 1. + add_offset: 0 + new_unit: "Percent" + obs_file: "CALIPSO_GOCCP_3.1.2_climo_200606-202012.nc" + obs_name: "CALIPSO" + obs_var_name: "CLDTOT_CAL" + category: "COSP" + +CLDHGH_CAL: + colormap: "cividis" + contour_levels_range: [0, 105, 5] + diff_colormap: "RdBu_r" + diff_contour_range: [-40, 40, 5] + scale_factor: 1. + add_offset: 0 + new_unit: "Percent" + obs_file: "CALIPSO_GOCCP_3.1.2_climo_200606-202012.nc" + obs_name: "CALIPSO" + obs_var_name: "CLDHGH_CAL" + category: "COSP" + +CLDMED_CAL: + colormap: "cividis" + contour_levels_range: [0, 105, 5] + diff_colormap: "RdBu_r" + diff_contour_range: [-40, 40, 5] + scale_factor: 1. + add_offset: 0 + new_unit: "Percent" + obs_file: "CALIPSO_GOCCP_3.1.2_climo_200606-202012.nc" + obs_name: "CALIPSO" + obs_var_name: "CLDMED_CAL" + category: "COSP" + +CLDLOW_CAL: + colormap: "cividis" + contour_levels_range: [0, 105, 5] + diff_colormap: "RdBu_r" + diff_contour_range: [-40, 40, 5] + scale_factor: 1. + add_offset: 0 + new_unit: "Percent" + obs_file: "CALIPSO_GOCCP_3.1.2_climo_200606-202012.nc" + obs_name: "CALIPSO" + obs_var_name: "CLDLOW_CAL" + category: "COSP" + +CLD_CAL: + colormap: "cividis" + contour_levels_range: [0, 105, 5] + diff_colormap: "RdBu_r" + diff_contour_range: [-40, 40, 5] + scale_factor: 1. + add_offset: 0 + new_unit: "Percent" + obs_file: "CALIPSO_GOCCP_3.1.2_climo_200606-202012.nc" + obs_name: "CALIPSO" + obs_var_name: "CLD_CAL" + category: "COSP" + #+++++++++++++++++ # Category: Other From 4986caae15fd63c7334f19393070f0f95292d0f3 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Thu, 28 Aug 2025 14:23:25 -0600 Subject: [PATCH 50/91] Fix incorrect stats in vector plot --- lib/plotting_functions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/plotting_functions.py b/lib/plotting_functions.py index e1ecdf935..4cd611c63 100644 --- a/lib/plotting_functions.py +++ b/lib/plotting_functions.py @@ -1122,9 +1122,9 @@ def plot_map_vect_and_save(wks, case_nickname, base_nickname, #Set stats: area_avg ax[0].set_title(f"Mean: {mdl_mag.weighted(wgt).mean().item():5.2f}\nMax: {mdl_mag.max():5.2f}\nMin: {mdl_mag.min():5.2f}", loc='right', fontsize=tiFontSize) - ax[1].set_title(f"Mean: {obs_mag.weighted(wgt).mean().item():5.2f}\nMax: {obs_mag.max():5.2f}\nMin: {mdl_mag.min():5.2f}", loc='right', + ax[1].set_title(f"Mean: {obs_mag.weighted(wgt).mean().item():5.2f}\nMax: {obs_mag.max():5.2f}\nMin: {obs_mag.min():5.2f}", loc='right', fontsize=tiFontSize) - ax[-1].set_title(f"Mean: {diff_mag.weighted(wgt).mean().item():5.2f}\nMax: {diff_mag.max():5.2f}\nMin: {mdl_mag.min():5.2f}", loc='right', + ax[-1].set_title(f"Mean: {diff_mag.weighted(wgt).mean().item():5.2f}\nMax: {diff_mag.max():5.2f}\nMin: {diff_mag.min():5.2f}", loc='right', fontsize=tiFontSize) # set rmse title: From f3b3caae09b848a4e8ca8026a5275d9d10f9889b Mon Sep 17 00:00:00 2001 From: Behrooz-Roozitalab Date: Wed, 3 Sep 2025 13:45:02 -0600 Subject: [PATCH 51/91] Add files to .gitignore --- .gitignore | 13 +- config_DCOTSS.yaml | 321 ++++ config_WACCM_beta06_WACCM_FWHIST.yaml | 583 +++++++ config_for_Simone_beta05.yaml | 474 ++++++ lib/adf_variable_defaults.yaml | 7 +- lib/adf_web.py | 8 +- scripts/analysis/aerosol_gas_tables.py | 243 ++- .../aerosol_gas_tables_Tropopause_version0.py | 1390 +++++++++++++++++ .../aerosol_gas_tables_Tropopause_version1.py | 1379 ++++++++++++++++ 9 files changed, 4332 insertions(+), 86 deletions(-) create mode 100644 config_DCOTSS.yaml create mode 100644 config_WACCM_beta06_WACCM_FWHIST.yaml create mode 100644 config_for_Simone_beta05.yaml create mode 100644 scripts/analysis/aerosol_gas_tables_Tropopause_version0.py create mode 100644 scripts/analysis/aerosol_gas_tables_Tropopause_version1.py diff --git a/.gitignore b/.gitignore index 0e4df2b8c..c1a1a63cb 100644 --- a/.gitignore +++ b/.gitignore @@ -109,4 +109,15 @@ GitHub.sublime-settings !.vscode/tasks.json !.vscode/launch.json !.vscode/extensions.json -.history \ No newline at end of file +.history + + +# ignore yaml files +*.yaml + +#ignore other local gas-aerosol files +scripts/analysis/aerosol_gas_tables_Tropopause_version0.py_Not_anymore! +scripts/analysis/aerosol_gas_tables_Tropopause_version1.py +scripts/analysis/aerosol_gas_tables_AllDefinitions.py +scripts/analysis/aerosol_gas_tables_ADF.py_withO3trop + diff --git a/config_DCOTSS.yaml b/config_DCOTSS.yaml new file mode 100644 index 000000000..0d8913f0a --- /dev/null +++ b/config_DCOTSS.yaml @@ -0,0 +1,321 @@ +#============================== +#config_cam_baseline_example.yaml + +#This is the main CAM diagnostics config file +#for doing comparisons of a CAM run against +#another CAM run, or a CAM baseline simulation. + +#Currently, if one is on NCAR's Casper or +#Cheyenne machine, then only the diagnostic output +#paths are needed, at least to perform a quick test +#run (these are indicated with "MUST EDIT" comments). +#Running these diagnostics on a different machine, +#or with a different, non-example simulation, will +#require additional modifications. +# +#Config file Keywords: +#-------------------- +# +#1. Using ${xxx} will substitute that text with the +# variable referenced by xxx. For example: +# +# cam_case_name: cool_run +# cam_climo_loc: /some/where/${cam_case_name} +# +# will set "cam_climo_loc" in the diagnostics package to: +# /some/where/cool_run +# +# Please note that currently this will only work if the +# variable only exists in one location in the file. +# +#2. Using ${.xxx} will do the same as +# keyword 1 above, but specifies which sub-section the +# variable is coming from, which is necessary for variables +# that are repeated in different subsections. For example: +# +# diag_basic_info: +# cam_climo_loc: /some/where/${diag_cam_climo.start_year} +# +# diag_cam_climo: +# start_year: 1850 +# +# will set "cam_climo_loc" in the diagnostics package to: +# /some/where/1850 +# +#Finally, please note that for both 1 and 2 the keywords must be lowercase. +#This is because future developments will hopefully use other keywords +#that are uppercase. Also please avoid using periods (".") in variable +#names, as this will likely cause issues with the current file parsing +#system. +#-------------------- +# +##============================== +# +# This file doesn't (yet) read environment variables, so the user must +# set this themselves. It is also a good idea to search the doc for 'user' +# to see what default paths are being set for output/working files. +# +# Note that the string 'USER-NAME-NOT-SET' is used in the jupyter script +# to check for a failure to customize +# +user: 'behroozr' + + +#This first set of variables specify basic info used by all diagnostic runs: +diag_basic_info: + + #Is this a model vs observations comparison? + #If "false" or missing, then a model-model comparison is assumed: + compare_obs: false + + #Generate HTML website (assumed false if missing): + #Note: The website files themselves will be located in the path + #specified by "cam_diag_plot_loc", under the "/website" subdirectory, + #where "" is the subdirectory created for this particular diagnostics run + #(usually "case_vs_obs_XXX" or "case_vs_baseline_XXX"). + create_html: false + + #Location of observational datasets: + #Note: this only matters if "compare_obs" is true and the path + #isn't specified in the variable defaults file. + obs_data_loc: /glade/campaign/cgd/amp/amwg/ADF_obs + + #Location where re-gridded and interpolated CAM climatology files are stored: + cam_regrid_loc: /glade/derecho/scratch/${user}/ADF/regrid + + #Overwrite CAM re-gridded files? + #If false, or missing, then regridding will be skipped for regridded variables + #that already exist in "cam_regrid_loc": + cam_overwrite_regrid: false + + #Location where diagnostic plots are stored: + cam_diag_plot_loc: /glade/derecho/scratch/${user}/ADF/plots + + #Location of ADF variable plotting defaults YAML file: + #If left blank or missing, ADF/lib/adf_variable_defaults.yaml will be used + #Uncomment and change path for custom variable defaults file + #defaults_file: /some/path/to/defaults/file.yaml + + #Vertical pressure levels (in hPa) on which to plot 3-D variables + #when using horizontal (e.g. lat/lon) map projections. + #If this config option is missing, then no 3-D variables will be plotted on + #horizontal maps. Please note too that pressure levels must currently match + #what is available in the observations file in order to be plotted in a + #model vs obs run: + plot_press_levels: [200,850] + + #Longitude line on which to center all lat/lon maps. + #If this config option is missing then the central + #longitude will default to 180 degrees E. + central_longitude: 180 + + #Number of processors on which to run the ADF. + #If this config variable isn't present then + #the ADF defaults to one processor. Also, if + #you set it to "*" then it will default + #to all of the processors available on a + #single node/machine: + num_procs: 8 + + #If set to true, then redo all plots even if they already exist. + #If set to false, then if a plot is found it will be skipped: + redo_plot: false + +#This second set of variables provides info for the CAM simulation(s) being diagnosed: +diag_cam_climo: + + # History file list of strings to match + # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] + # Only affects timeseries as everything else uses the created timeseries + # Default: + hist_str: cam.hm + + #Calculate climatologies? + #If false, the climatology files will not be created: + calc_cam_climo: false + + #Overwrite CAM climatology files? + #If false, or not prsent, then already existing climatology files will be skipped: + cam_overwrite_climo: false + + #Name of CAM case (or CAM run name): + cam_case_name: f.e22.FCnudged.f09_32L.slh_released.2019.DCOTSS_ACCLIP.finn.cams6Mosaic.v01 + + #Case nickname + #NOTE: if nickname starts with '0' - nickname must be in quotes! + # ie '026a' as opposed to 026a + #If missing or left blank, will default to cam_case_name + case_nickname: DCOTSS_1deg_2022 #cool nickname + + #Location of CAM history (h0) files: + #Example test files + cam_hist_loc: /glade/campaign/acom/acom-weather/behroozr/NASA_DCOTSS/cases2024/${diag_cam_climo.cam_case_name}/atm/hist/ + + #Location of CAM climatologies (to be created and then used by this script) + cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/climo + + #model year when time series files should start: + #Note: Leaving this entry blank will make time series + # start at earliest available year. + start_year: 2022 #10 + + #model year when time series files should end: + #Note: Leaving this entry blank will make time series + # end at latest available year. + end_year: 2022 #14 + + #Do time series files exist? + #If True, then diagnostics assumes that model files are already time series. + #If False, or if simply not present, then diagnostics will attempt to create + #time series files from history (time-slice) files: + cam_ts_done: false + + #Save interim time series files? + #WARNING: This can take up a significant amount of space, + # but will save processing time the next time + cam_ts_save: false + + #Overwrite time series files, if found? + #If set to false, then time series creation will be skipped if files are found: + cam_overwrite_ts: false + + #Location where time series files are (or will be) stored: + cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/ts + + + +#This third set of variables provide info for the CAM baseline climatologies. +#This only matters if "compare_obs" is false: +diag_cam_baseline_climo: + + # History file list of strings to match + # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] + # Only affects timeseries as everything else uses the created timeseries + # Default: + hist_str: cam.h0 + + #Calculate cam baseline climatologies? + #If false, the climatology files will not be created: + calc_cam_climo: false + + #Overwrite CAM climatology files? + #If false, or not present, then already existing climatology files will be skipped: + cam_overwrite_climo: false + + #Name of CAM baseline case: + cam_case_name: f.e22.FCnudged.f09_32L.slh_released.2019.DCOTSS_ACCLIP.finn.cams6Mosaic.v01 + + #Baseline case nickname + #NOTE: if nickname starts with '0' - nickname must be in quotes! + # ie '026a' as opposed to 026a + #If missing or left blank, will default to cam_case_name + case_nickname: DCOTSS_1deg_2021 #cool nickname + + #Location of CAM baseline history (h0) files: + #Example test files + cam_hist_loc: /glade/campaign/acom/acom-weather/behroozr/NASA_DCOTSS/cases2024/${diag_cam_baseline_climo.cam_case_name}/atm/hist/ + + #Location of baseline CAM climatologies: + cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/climo + + #model year when time series files should start: + #Note: Leaving this entry blank will make time series + # start at earliest available year. + start_year: 2021 #10 + + #model year when time series files should end: + #Note: Leaving this entry blank will make time series + # end at latest available year. + end_year: 2021 #14 + + #Do time series files need to be generated? + #If True, then diagnostics assumes that model files are already time series. + #If False, or if simply not present, then diagnostics will attempt to create + #time series files from history (time-slice) files: + cam_ts_done: false + + #Save interim time series files for baseline run? + #WARNING: This can take up a significant amount of space: + cam_ts_save: false + + #Overwrite baseline time series files, if found? + #If set to false, then time series creation will be skipped if files are found: + cam_overwrite_ts: false + + #Location where time series files are (or will be) stored: + cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/ts + + + +#+++++++++++++++++++++++++++++++++++++++++++++++++++ +#These variables below only matter if you are using +#a non-standard method, or are adding your own +#diagnostic scripts. +#+++++++++++++++++++++++++++++++++++++++++++++++++++ + +#Note: If you want to pass arguments to a particular script, you can +#do it like so (using the "averaging_example" script in this case): +# - {create_climo_files: {kwargs: {clobber: true}}} + +#Name of time-averaging scripts being used to generate climatologies. +#These scripts must be located in "scripts/averaging": +time_averaging_scripts: + - create_climo_files + +#Name of regridding scripts being used. +#These scripts must be located in "scripts/regridding": +regridding_scripts: + - regrid_and_vert_interp + +#List of analysis scripts being used. +#These scripts must be located in "scripts/analysis": +analysis_scripts: + - aerosol_gas_tables + +#List of plotting scripts being used. +#These scripts must be located in "scripts/plotting": +plotting_scripts: + # - global_latlon_map + # - global_latlon_vect_map + # - zonal_mean + # - meridional_mean + # - ozone_diagnostics + +#List of CAM variables that will be processesd: +#If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed +diag_var_list: + - CO + #- tr_CH2CL2_AS + # - CH2CL2 + # - O3 + #- AODVISdn + #- BC + + # - SWCF + #- LWCF + #- PRECC + #- PRECL + #- PSL + #- Q + #- U + #- T + #- RELHUM + #- TREFHT + #- TS + #- TAUX + #- TAUY + #- FSNT + #- FLNT + #- RESTOM + # - AODVISdn + #- Q + #- BC + #- POM + #- SO4 + #- SOA + #- DUST + #- SeaSalt + #- O3 + + +#END OF FILE diff --git a/config_WACCM_beta06_WACCM_FWHIST.yaml b/config_WACCM_beta06_WACCM_FWHIST.yaml new file mode 100644 index 000000000..722c669f6 --- /dev/null +++ b/config_WACCM_beta06_WACCM_FWHIST.yaml @@ -0,0 +1,583 @@ +#============================== +#config_cam_baseline_example.yaml + +#This is the main CAM diagnostics config file +#for doing comparisons of a CAM run against +#another CAM run, or a CAM baseline simulation. + +#Currently, if one is on NCAR's Casper or +#Cheyenne machine, then only the diagnostic output +#paths are needed, at least to perform a quick test +#run (these are indicated with "MUST EDIT" comments). +#Running these diagnostics on a different machine, +#or with a different, non-example simulation, will +#require additional modifications. +# +#Config file Keywords: +#-------------------- +# +#1. Using ${xxx} will substitute that text with the +# variable referenced by xxx. For example: +# +# cam_case_name: cool_run +# cam_climo_loc: /some/where/${cam_case_name} +# +# will set "cam_climo_loc" in the diagnostics package to: +# /some/where/cool_run +# +# Please note that currently this will only work if the +# variable only exists in one location in the file. +# +#2. Using ${.xxx} will do the same as +# keyword 1 above, but specifies which sub-section the +# variable is coming from, which is necessary for variables +# that are repeated in different subsections. For example: +# +# diag_basic_info: +# cam_climo_loc: /some/where/${diag_cam_climo.start_year} +# +# diag_cam_climo: +# start_year: 1850 +# +# will set "cam_climo_loc" in the diagnostics package to: +# /some/where/1850 +# +#Finally, please note that for both 1 and 2 the keywords must be lowercase. +#This is because future developments will hopefully use other keywords +#that are uppercase. Also please avoid using periods (".") in variable +#names, as this will likely cause issues with the current file parsing +#system. +#-------------------- +# +##============================== +# +# This file doesn't (yet) read environment variables, so the user must +# set this themselves. It is also a good idea to search the doc for 'user' +# to see what default paths are being set for output/working files. +# +# Note that the string 'USER-NAME-NOT-SET' is used in the jupyter script +# to check for a failure to customize +# +user: 'behroozr' + + +#This first set of variables specify basic info used by all diagnostic runs: +diag_basic_info: + + #Is this a model vs observations comparison? + #If "false" or missing, then a model-model comparison is assumed: + compare_obs: false + + #Generate HTML website (assumed false if missing): + #Note: The website files themselves will be located in the path + #specified by "cam_diag_plot_loc", under the "/website" subdirectory, + #where "" is the subdirectory created for this particular diagnostics run + #(usually "case_vs_obs_XXX" or "case_vs_baseline_XXX"). + create_html: true + + #Location of observational datasets: + #Note: this only matters if "compare_obs" is true and the path + #isn't specified in the variable defaults file. + obs_data_loc: /glade/campaign/cgd/amp/amwg/ADF_obs + + #Location where re-gridded and interpolated CAM climatology files are stored: + cam_regrid_loc: /glade/derecho/scratch/${user}/ADF/regrid + + #Overwrite CAM re-gridded files? + #If false, or missing, then regridding will be skipped for regridded variables + #that already exist in "cam_regrid_loc": + cam_overwrite_regrid: false + + #Location where diagnostic plots are stored: + cam_diag_plot_loc: /glade/derecho/scratch/${user}/ADF/plots + + #Location of ADF variable plotting defaults YAML file: + #If left blank or missing, ADF/lib/adf_variable_defaults.yaml will be used + #Uncomment and change path for custom variable defaults file + #defaults_file: /some/path/to/defaults/file.yaml + + #Vertical pressure levels (in hPa) on which to plot 3-D variables + #when using horizontal (e.g. lat/lon) map projections. + #If this config option is missing, then no 3-D variables will be plotted on + #horizontal maps. Please note too that pressure levels must currently match + #what is available in the observations file in order to be plotted in a + #model vs obs run: + plot_press_levels: [200,850] + + #Longitude line on which to center all lat/lon maps. + #If this config option is missing then the central + #longitude will default to 180 degrees E. + central_longitude: 180 + + #Number of processors on which to run the ADF. + #If this config variable isn't present then + #the ADF defaults to one processor. Also, if + #you set it to "*" then it will default + #to all of the processors available on a + #single node/machine: + num_procs: 8 + + #If set to true, then redo all plots even if they already exist. + #If set to false, then if a plot is found it will be skipped: + redo_plot: false + +#This second set of variables provides info for the CAM simulation(s) being diagnosed: +diag_cam_climo: + + # History file list of strings to match + # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] + # Only affects timeseries as everything else uses the created timeseries + # Default: + hist_str: cam.h0a + #hist_str: cam.hm + + #Calculate climatologies? + #If false, the climatology files will not be created: + calc_cam_climo: false + + #Overwrite CAM climatology files? + #If false, or not prsent, then already existing climatology files will be skipped: + cam_overwrite_climo: false + + #Name of CAM case (or CAM run name): + cam_case_name: f.e30_beta06_megan.FWHIST_f09_f09_mg17v1.L70.cam6.clm6.002 + + #Case nickname + #NOTE: if nickname starts with '0' - nickname must be in quotes! + # ie '026a' as opposed to 026a + #If missing or left blank, will default to cam_case_name + case_nickname: "WACCM (beta06)" + + #Location of CAM history (h0) files: + #Example test files + cam_hist_loc: /glade/derecho/scratch/shawnh/archive/f.e30_beta06_megan.FWHIST_f09_f09_mg17v1.L70.cam6.clm6.002/atm/hist + #cam_hist_loc: /glade/derecho/scratch/behroozr/Budgets + + #Location of CAM climatologies (to be created and then used by this script) + cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/climo + + #model year when time series files should start: + #Note: Leaving this entry blank will make time series + # start at earliest available year. + start_year: 2010 + + #model year when time series files should end: + #Note: Leaving this entry blank will make time series + # end at latest available year. + end_year: 2010 + + #Do time series files exist? + #If True, then diagnostics assumes that model files are already time series. + #If False, or if simply not present, then diagnostics will attempt to create + #time series files from history (time-slice) files: + cam_ts_done: false + + #Save interim time series files? + #WARNING: This can take up a significant amount of space, + # but will save processing time the next time + cam_ts_save: true + + #Overwrite time series files, if found? + #If set to false, then time series creation will be skipped if files are found: + cam_overwrite_ts: false + + #Location where time series files are (or will be) stored: + cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/ts + + #TEM diagnostics + #--------------- + #TEM history file number + #If missing or blank, ADF will default to h4 + tem_hist_str: cam.h4a + + #Location where TEM files are stored: + #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! + cam_tem_loc: /glade/derecho/scratch/${user}/${diag_cam_climo.cam_case_name}/tem/ + + #Overwrite TEM files, if found? + #If set to false, then TEM creation will be skipped if files are found: + overwrite_tem: false + + #---------------------- + + #You can alternatively provide a list of cases, which will make the ADF + #apply the same diagnostics to each case separately in a single ADF session. + #All of the config variables below show how it is done, and are the only ones + #that need to be lists. This also automatically enables the generation of + #a "main_website" in "cam_diag_plot_loc" that brings all of the different cases + #together under a single website. + + #Also please note that config keywords cannot currently be used in list mode. + + #cam_case_name: + # - b.e23_alpha17f.BLT1850.ne30_t232.098 + # - b.e23_alpha17f.BLT1850.ne30_t232.095 + + #Case nickname + #NOTE: if nickname starts with '0' - nickname must be in quotes! + # ie '026a' as opposed to 026a + #If missing or left blank, will default to cam_case_name + #case_nickname: + # - cool nickname + # - cool nickname 2 + + #calc_cam_climo: + # - true + # - true + + #cam_overwrite_climo: + # - false + # - false + + #cam_hist_loc: + # - /glade/campaign/cgd/amp/amwg/ADF_test_cases/b.e23_alpha17f.BLT1850.ne30_t232.098 + # - /glade/campaign/cgd/amp/amwg/ADF_test_cases/b.e23_alpha17f.BLT1850.ne30_t232.095 + + #cam_climo_loc: + # - /some/where/you/want/to/have/climo_files/ #MUST EDIT! + # - /the/same/or/some/other/climo/files/location + + #start_year: + # - 10 + # - 10 + + #end_year: + # - 14 + # - 14 + + #cam_ts_done: + # - false + # - false + + #cam_ts_save: + # - true + # - true + + #cam_overwrite_ts: + # - false + # - false + + #cam_ts_loc: + # - /some/where/you/want/to/have/time_series_files + # - /same/or/different/place/you/want/files + + #TEM diagnostics + #--------------- + #TEM history file number + #If missing or blank, ADF will default to h4 + #tem_hist_str: + # - cam.h4 + # - cam.h# + + #Location where TEM files are stored: + #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! + #cam_tem_loc: + # - /some/where/you/want/to/have/TEM_files/ + # - /same/or/different/place/you/want/TEM_files/ + + #Overwrite TEM files, if found? + #If set to false, then TEM creation will be skipped if files are found: + #overwrite_tem: + # - false + # - true + + #---------------------- + + +#This third set of variables provide info for the CAM baseline climatologies. +#This only matters if "compare_obs" is false: +diag_cam_baseline_climo: + + # History file list of strings to match + # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] + # Only affects timeseries as everything else uses the created timeseries + # Default: + hist_str: cam.h0 + + #Calculate cam baseline climatologies? + #If false, the climatology files will not be created: + calc_cam_climo: false + + #Overwrite CAM climatology files? + #If false, or not present, then already existing climatology files will be skipped: + cam_overwrite_climo: false + + #Name of CAM baseline case: + cam_case_name: f.e22.FWHISTnudged.f09_f09.cesm2.2.0.2001-2021.001 + + #Baseline case nickname + #NOTE: if nickname starts with '0' - nickname must be in quotes! + # ie '026a' as opposed to 026a + #If missing or left blank, will default to cam_case_name + case_nickname: "WACCM (DOUGK)" #cool nickname + + #Location of CAM baseline history (h0) files: + #Example test files + cam_hist_loc: /glade/campaign/acom/acom-climate/UTLS/shawnh/archive/f.e22.FWHISTnudged.f09_f09.cesm2.2.0.2001-2021.001/atm/hist + + #Location of baseline CAM climatologies: + cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/climo + + #model year when time series files should start: + #Note: Leaving this entry blank will make time series + # start at earliest available year. + start_year: 2010 + + #model year when time series files should end: + #Note: Leaving this entry blank will make time series + # end at latest available year. + end_year: 2010 + + #Do time series files need to be generated? + #If True, then diagnostics assumes that model files are already time series. + #If False, or if simply not present, then diagnostics will attempt to create + #time series files from history (time-slice) files: + cam_ts_done: false + + #Save interim time series files for baseline run? + #WARNING: This can take up a significant amount of space: + cam_ts_save: true + + #Overwrite baseline time series files, if found? + #If set to false, then time series creation will be skipped if files are found: + cam_overwrite_ts: false + + #Location where time series files are (or will be) stored: + cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/ts + + #TEM diagnostics + #--------------- + #TEM history file number + #If missing or blank, ADF will default to h4 + tem_hist_str: cam.h4a + + #Location where TEM files are stored: + #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! + cam_tem_loc: /glade/derecho/scratch/${user}/${diag_cam_baseline_climo.cam_case_name}/tem/ + + #Overwrite TEM files, if found? + #If set to false, then TEM creation will be skipped if files are found: + overwrite_tem: false + + +#This fourth set of variables provides settings for calling the Climate Variability +# Diagnostics Package (CVDP). If cvdp_run is set to true the CVDP will be set up and +# run in background mode, likely completing after the ADF has completed. +# If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed +# in the diag_var_list variable listing. +# For more CVDP information: https://www.cesm.ucar.edu/working_groups/CVC/cvdp/ +diag_cvdp_info: + + # Run the CVDP on the listed run(s)? + cvdp_run: false + + # CVDP code path, sets the location of the CVDP codebase + # CGD systems path = /home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ + # CISL systems path = /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ + # github location = https://github.com/NCAR/CVDP-ncl + cvdp_codebase_loc: /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ + + # Location where cvdp codebase will be copied to and diagnostic plots will be stored + cvdp_loc: /glade/derecho/scratch/${user}/ADF/cvdp/ + + # tar up CVDP results? + cvdp_tar: false + +# This set of variables provides settings for calling NOAA's +# Model Diagnostic Task Force (MDTF) diagnostic package. +# https://github.com/NOAA-GFDL/MDTF-diagnostics +# +# If mdtf_run: true, the MDTF will be set up and +# run in background mode, likely completing after the ADF has completed. +# +# WARNING: This currently only runs on CASPER (not derecho) +# +# The variables required depend on the diagnostics (PODs) selected. +# AMWG-developed PODS and their required variables: +# (Note that PRECT can be computed from PRECC & PRECL) +# - MJO_suite: daily PRECT, FLUT, U850, U200, V200 (all required) +# - Wheeler-Kiladis Wavenumber Frequency Spectra: daily PRECT, FLUT, U200, U850, OMEGA500 +# (will use what is available) +# - Blocking (Rich Neale): daily OMEGA500 +# - Precip Diurnal Cycle (Rich Neale): 3-hrly PRECT +# +# Many other diagnostics are available; see +# https://mdtf-diagnostics.readthedocs.io/en/main/sphinx/start_overview.html + +# +diag_mdtf_info: + # Run the MDTF on the model cases + mdtf_run: false + + # The file that will be written by ADF to input to MDTF. Call this whatever you want. + mdtf_input_settings_filename : mdtf_input.json + + ## MDTF code path, sets the location of the MDTF codebase and pre-compiled conda envs + # CHANGE if you have any: your own MDTF code, installed conda envs and/or obs_data + + mdtf_codebase_path : /glade/campaign/cgd/amp/amwg/mdtf + mdtf_codebase_loc : ${mdtf_codebase_path}/MDTF-diagnostics.v3.1.20230817.ADF + conda_root : /glade/u/apps/opt/conda + conda_env_root : ${mdtf_codebase_path}/miniconda2/envs.MDTFv3.1.20230412/ + OBS_DATA_ROOT : ${mdtf_codebase_path}/obs_data + + # SET this to a writable dir. The ADF will place ts files here for the MDTF to read (adds the casename) + MODEL_DATA_ROOT : ${diag_cam_climo.cam_ts_loc}/mdtf/inputdata/model + + # Choose diagnostics (PODs). Full list of available PODs: https://github.com/NOAA-GFDL/MDTF-diagnostics + pod_list : [ "MJO_suite" ] + + # Intermediate/output file settings + make_variab_tar: false # tar up MDTF results + save_ps : false # save postscript figures in addition to bitmaps + save_nc : false # save netCDF files of processed data (recommend true when starting with new model data) + overwrite: true # overwrite results in OUTPUT_DIR; otherwise results will be saved under a unique name + + # Settings used in debugging: + verbose : 3 # Log verbosity level. + test_mode: false # Set to true for framework test. Data is fetched but PODs are not run. + dry_run : false # Framework test. No external commands are run and no remote data is copied. Implies test_mode. + + # Settings that shouldn't change in ADF implementation for now + data_type : single_run # single_run or multi_run (only works with single right now) + data_manager : Local_File # Fetch data or it is local? + environment_manager : Conda # Manage dependencies + + + +#+++++++++++++++++++++++++++++++++++++++++++++++++++ +#These variables below only matter if you are using +#a non-standard method, or are adding your own +#diagnostic scripts. +#+++++++++++++++++++++++++++++++++++++++++++++++++++ + +#Note: If you want to pass arguments to a particular script, you can +#do it like so (using the "averaging_example" script in this case): +# - {create_climo_files: {kwargs: {clobber: true}}} + +#Name of time-averaging scripts being used to generate climatologies. +#These scripts must be located in "scripts/averaging": +#time_averaging_scripts: +# - create_climo_files + #- create_TEM_files #To generate TEM files, please un-comment + +#Name of regridding scripts being used. +#These scripts must be located in "scripts/regridding": +#regridding_scripts: +# - regrid_and_vert_interp + +#List of analysis scripts being used. +#These scripts must be located in "scripts/analysis": +analysis_scripts: + # - amwg_table + - aerosol_gas_tables + +#List of plotting scripts being used. +#These scripts must be located in "scripts/plotting": +#plotting_scripts: +# - global_latlon_map +# - global_latlon_vect_map +# - zonal_mean +# - meridional_mean +# - polar_map +# - cam_taylor_diagram +# - qbo +# - ozone_diagnostics +# - tape_recorder + #- MOPITT +# - seasonal_cycle + #- tem + #- regional_map_multicase #To use this please un-comment and fill-out + #the "region_multicase" section below + +#List of CAM variables that will be processesd: +#If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed +diag_var_list: + - SWCF +# - LWCF +# - PRECC +# - PRECL +# - PSL +# - Q +# - U +# - T +# - RELHUM +# - TREFHT +# - TS +# - TAUX +# - TAUY +# - FSNT +# - FLNT +# - LANDFRAC +# - O3 +# - O3S +# - CO +# - CO2 +# - H2O +# - NOX +# - NOY +# - CLDICE +# - DMS +# - EXTINCTdn +# - CFC11 +# - N2O +# - HNO3 +# - ISOP +# - CH4 +# - OH +# - SAD_TROP +# - SAD_AERO +# - SAD_SULFC +# - LNO_PROD +# - bc_a1 +# - bc_a4 +# - SO2 +# - dst_a1 +# - dst_a2 +# - dst_a3 +# - ncl_a1 +# - ncl_a2 +# - ncl_a3 +# - num_a1 +# - num_a2 +# - num_a3 +# - num_a4 +# - num_a5 +# - pom_a1 +# - pom_a4 +# - so4_a1 +# - so4_a2 +# - so4_a3 +# - so4_a5 +# - FLASHFRQ +# - LNO_COL_PROD +## - AODDUST +# - AODVIS +# - AODVISdn +# - MEG_ISOP + +# +# MDTF recommended variables +# - FLUT +# - OMEGA500 +# - PRECT +# - PS +# - PSL +# - U200 +# - U850 +# - V200 +# - V850 + +# Options for multi-case regional contour plots (./plotting/regional_map_multicase.py) +# region_multicase: +# region_spec: [slat, nlat, wlon, elon] +# region_time_option: # If calendar, will look for specified years. If zeroanchor will use a nyears starting from year_offset from the beginning of timeseries +# region_start_year: +# region_end_year: +# region_nyear: +# region_year_offset: +# region_month: +# region_season: +# region_variables: + +#END OF FILE diff --git a/config_for_Simone_beta05.yaml b/config_for_Simone_beta05.yaml new file mode 100644 index 000000000..fce6d7b79 --- /dev/null +++ b/config_for_Simone_beta05.yaml @@ -0,0 +1,474 @@ +#============================== +#config_cam_baseline.yaml + +#This is the main CAM diagnostics config file +#for doing comparisons of a CAM run against +#another CAM run, or a CAM baseline simulation. + +#Currently, if one is on NCAR's Casper or +#Cheyenne machine, then only the diagnostic output +#paths are needed, at least to perform a quick test +#run (these are indicated with "MUST EDIT" comments). +#Running these diagnostics on a different machine, +#or with a different, non-example simulation, will +#require additional modifications. +# +#Config file Keywords: +#-------------------- +# +#1. Using ${xxx} will substitute that text with the +# variable referenced by xxx. For example: +# +# cam_case_name: cool_run +# cam_climo_loc: /some/where/${cam_case_name} +# +# will set "cam_climo_loc" in the diagnostics package to: +# /some/where/cool_run +# +# Please note that currently this will only work if the +# variable only exists in one location in the file. +# +#2. Using ${.xxx} will do the same as +# keyword 1 above, but specifies which sub-section the +# variable is coming from, which is necessary for variables +# that are repeated in different subsections. For example: +# +# diag_basic_info: +# cam_climo_loc: /some/where/${diag_cam_climo.start_year} +# +# diag_cam_climo: +# start_year: 1850 +# +# will set "cam_climo_loc" in the diagnostics package to: +# /some/where/1850 +# +#Finally, please note that for both 1 and 2 the keywords must be lowercase. +#This is because future developments will hopefully use other keywords +#that are uppercase. Also please avoid using periods (".") in variable +#names, as this will likely cause issues with the current file parsing +#system. +#-------------------- +# +##============================== + +#diag_loc: /glade/scratch/richling/adf-output/f.cam6_3_132.FMTHIST_ne30.taubgnd5.energy_front_off_rd_beta_1_vs_f.cam6_3_119.FMTHIST_ne30.r328_gamma0.33_soae.nudged_dst11.001/ + +#climo_loc: /glade/scratch/richling/adf-output/ADF-data/climos/ #/glade/campaign/cgd/amp/amwg/climo/ +#ts_loc: /glade/scratch/richling/adf-output/ADF-data/timeseries/ + +user: 'behroozr' + +#This first set of variables specify basic info used by all diagnostic runs: +diag_basic_info: + # diag_loc: /glade/derecho/scratch/richling/adf-output/${diag_cam_climo.cam_case_name}_vs_${diag_cam_baseline_climo.cam_case_name}/ + + # climo_loc: /glade/derecho/scratch/richling/adf-output/ADF-data/climo/ + # ts_loc: /glade/derecho/scratch/richling/adf-output/ADF-data/timeseries/ + + diag_loc: /glade/derecho/scratch/behroozr/Budgets/output_test/${diag_cam_climo.cam_case_name}_vs_${diag_cam_baseline_climo.cam_case_name}/ + + climo_loc: /glade/derecho/scratch/behroozr/Budgets//output_test/ADF-data/climo/ + ts_loc: /glade/derecho/scratch/behroozr/Budgets/output_test/ADF-data/timeseries/ + + + #History file string to match (eg. cam.h0 or ocn.pop.h.ecosys.nday1) + # Only affects timeseries as everything else uses timeseries + # Leave off trailing '.' + #Default: cam.h0 + #hist_str: cam.h0a + + #Is this a model vs observations comparison? + #If "false" or missing, then a model-model comparison is assumed: + compare_obs: false + + #Generate HTML website (assumed false if missing): + #Note: The website files themselves will be located in the path + #specified by "cam_diag_plot_loc", under the "/website" subdirectory, + #where "" is the subdirectory created for this particular diagnostics run + #(usually "case_vs_obs_XXX" or "case_vs_baseline_XXX"). + create_html: true + + #Location of observational datasets: + #Note: this only matters if "compare_obs" is true and the path + #isn't specified in the variable defaults file. + obs_data_loc: /glade/campaign/cgd/amp/amwg/ADF_obs + + #Location where re-gridded CAM climatology files are stored: + cam_regrid_loc: ${diag_loc}regrid/ + + #Overwrite CAM re-gridded files? + #If false, or missing, then regridding will be skipped for regridded variables + #that already exist in "cam_regrid_loc": + cam_overwrite_regrid: false + + #Location where diagnostic plots are stored: + cam_diag_plot_loc: ${diag_loc}diag-plot/ + + #Use default variable plot settings? + #If "true", then variable-specific plotting attributes as defined in + #ADF/lib/adf_variable_defaults.yaml will be used: + use_defaults: true + + #Location of ADF variable plotting defaults YAML file + #if not using the one in ADF/lib: + #defaults_file: /some/path/to/defaults/file + + #Vertical pressure levels (in hPa) on which to plot 3-D variables + #when using horizontal (e.g. lat/lon) map projections. + #If this config option is missing, then no 3-D variables will be plotted on + #horizontal maps: + plot_press_levels: [200,850] + + #Apply monthly weights to seasonal averages. + #If False or missing, then all months are + #given the same weight: + weight_season: True + + num_procs: 8 + + redo_plot: false + + + + + + + + + +#This second set of variables provides info for the CAM simulation(s) being diagnosed: +diag_cam_climo: + + #Calculate climatologies? + #If false, neither the climatology or time-series files will be created: + calc_cam_climo: true + + #Overwrite CAM climatology files? + #If false, or not prsent, then already existing climatology files will be skipped: + cam_overwrite_climo: false + + #Name of CAM case (or CAM run name): + #cam_case_name: f.cam6_3_153.FCMTnudged_climate_chemistry_ne30.factor_fix + # case_nickname: f.cam6_3_153.FCMTnudged_climate_chemistry_ne30.factor_fix + + #cam_case_name: f.cam6_3_160.FCMT_ne30.moving_mtn.002 + #case_nickname: f.cam6_3_160.FCMT_ne30.moving_mtn.002 + #start_year: 1996 + #end_year: 1997 + + hist_str: cam.h0a + #hist_str: cam.h0 + + + # cam_case_name: FCnudged_f09.mam.Jul9.1995_2020.001 + # case_nickname: FCnudged_f09.mam.Jul9.1995_2020.001 + # start_year: 2002 + # end_year: 2019 + + + # cam_case_name: FCnudged_f09.mam.mar27.2000_2021.001 + # case_nickname: FCnudged_f09.mam.mar27.2000_2021.001 + # start_year: 2002 + # end_year: 2019 + + # cam_case_name: b.e22.BWSSP585cmip6ctsslh.f09_f09_mg17.cesm2.2_vsl03.present.CMIP_MLtracers_tagged.Halogensv002 + # case_nickname: B_585P_ML + # start_year: 2016 + # end_year: 2025 + + # cam_case_name: b.e22.BWSSP585cmip6ctsslh.f09_f09_mg17.cesm2.2_vsl03.future.CMIP_MLtracers_tagged.Halogensv002 + # case_nickname: B_585F_ML + # start_year: 2090 + # end_year: 2099 + + # cam_case_name: b.e22.BWSSP585cmip6ctsslh.f09_f09_mg17.cesm2.2_vsl03.future.CMIP_MLtracers_tagged.Halogensv001 + # case_nickname: B_585F_Ord + # start_year: 2090 + # end_year: 2099 + + cam_case_name: b.e22.BWSSP585cmip6ctsslh.f09_f09_mg17.cesm2.2_vsl03.future.CMIP_MLtracers_tagged.Halogensv001 + case_nickname: B_585P_Ord + start_year: 2016 + end_year: 2025 + + cam_case_name: f.e30_beta05.FCts4MTHIST.ne30_L93.cmip7.001 + case_nickname: beta_05 + start_year: 1980 + end_year: 1980 + + # cam_case_name: f.e22.FCnudged.f09_f09_mg17.slh.2000.fire.ct.finn.mosaic_cams6.001 + # case_nickname: Ben_FV + # start_year: 2003 + # end_year: 2003 + + # cam_case_name: f.e30_alpha04a.FCts4MTHIST.ne30_L93.cmip7_old_volc + # case_nickname: beta_04 + # start_year: 1980 + # end_year: 1980 + + # #cam_case_name: f.cam6_3_160.FMTHIST_ne30.moving_mtn.output.001 + # #case_nickname: f.cam6_3_160.FMTHIST_ne30.moving_mtn.output.001 + # #start_year: 1996 + # #end_year: 2001 + + + + + + # #Location of CAM history (h0) files: + # #/glade/derecho/scratch/tilmes/archive/f.cam6_3_153.FCMTnudged_climate_chemistry_ne30.001/atm/hist + # #cam_hist_loc: /glade/derecho/scratch/tilmes/archive/${diag_cam_climo.cam_case_name}/atm/hist/ + #cam_hist_loc: /glade/campaign/acom/acom-climate/UTLS/shawnh/archive/${diag_cam_climo.cam_case_name}/atm/hist/ + #cam_hist_loc: /glade/campaign/acom/acom-weather/behroozr/VSLS/cases2024//${diag_cam_climo.cam_case_name}/atm/hist/ + cam_hist_loc: /glade/derecho/scratch/shawnh/archive//${diag_cam_climo.cam_case_name}/atm/hist/ + #cam_hist_loc: /glade/campaign/acom/acom-climate/UTLS/shawnh/archive/${diag_cam_climo.cam_case_name}/atm/hist/ + #cam_hist_loc: /glade/campaign/acom/acom-da/Methane_simulations/f.e22.FCnudged.f09_f09_mg17.slh.2000.fire.ct.finn.mosaic_cams6.001/H0/ + + # cam_case_name: f.e22.FHIST.ne0np4.India07.ne30x1_ne30x1_mt12_cesm2.2_rel_bugFixed_noPATC + # case_nickname: f.e22.FHIST.ne0np4.India07.ne30x1_ne30x1_mt12_cesm2.2_rel_bugFixed_noPATC + # start_year: 2002 + # end_year: 2002 + # cam_hist_loc: /glade/campaign/acom/acom-weather/behroozr/India_Dust/Simulation2_${diag_cam_climo.cam_case_name}/atm/hist/ + + + #Location of CAM climatologies: + cam_climo_loc: ${climo_loc}${diag_cam_climo.cam_case_name}/yrs_1995_2001/ + + #model year when time series files should start: + #Note: Leaving this entry blank will make time series + # start at earliest available year. + #start_year: 2001 + + #model year when time series files should end: + #Note: Leaving this entry blank will make time series + # end at latest available year. + #end_year: 2001 + + #Do time series files need to be generated? + #If True, then diagnostics assumes that model files are already time series. + #If False, or if simply not present, then diagnostics will attempt to create + #time series files from history (time-slice) files: + cam_ts_done: false + + #Save interim time series files? + #WARNING: This can take up a significant amount of space: + cam_ts_save: true + + #Overwrite time series files, if found? + #If set to false, then time series creation will be skipped if files are found: + cam_overwrite_ts: false + + #Location where time series files are (or will be) stored: + cam_ts_loc: ${ts_loc}${diag_cam_climo.cam_case_name}/yrs_1995_2001/ + + #TEM diagnostics + #--------------- + #TEM history file number + #If missing or blank, ADF will default to h4 + tem_hist_str: cam.h4 + + #Location where TEM files are stored: + #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! + cam_tem_loc: /glade/scratch/${user}/${diag_cam_climo.cam_case_name}/tem/ + + #Overwrite TEM files, if found? + #If set to false, then TEM creation will be skipped if files are found: + overwrite_tem: false + + +#This third set of variables provide info for the CAM baseline climatologies. +#This only matters if "compare_obs" is false: +diag_cam_baseline_climo: + + #Calculate cam baseline climatologies? + #If false, neither the climatology or time-series files will be created: + calc_cam_climo: true + + #Overwrite CAM climatology files? + #If false, or not present, then already existing climatology files will be skipped: + cam_overwrite_climo: false + #hist_str: cam.h0a + hist_str: cam.h0 + + #Name of CAM baseline case: + cam_case_name: FCnudged_f09.mam.mar27.2000_2021.001 + + case_nickname: FCnudged_f09.mam.mar27.2000_2021.001 + + #Location of CAM baseline history (h0) files: + #/glade/derecho/scratch/tilmes/archive/f.cam6_3_153.FCMTnudged_ne30.001/atm/hist + #cam_hist_loc: /glade/derecho/scratch/tilmes/archive/${diag_cam_baseline_climo.cam_case_name}/atm/hist/ + cam_hist_loc: /glade/campaign/acom/acom-climate/UTLS/shawnh/archive/${diag_cam_baseline_climo.cam_case_name}/atm/hist/ + + + #Location of baseline CAM climatologies: + cam_climo_loc: ${climo_loc}${diag_cam_baseline_climo.cam_case_name}/yrs_1995_2001/ + + #model year when time series files should start: + #Note: Leaving this entry blank will make time series + # start at earliest available year. + start_year: 2002 + + #model year when time series files should end: + #Note: Leaving this entry blank will make time series + # end at latest available year. + end_year: 2002 + + #Do time series files need to be generated? + #If True, then diagnostics assumes that model files are already time series. + #If False, or if simply not present, then diagnostics will attempt to create + #time series files from history (time-slice) files: + cam_ts_done: false + + #Save interim time series files for baseline run? + #WARNING: This can take up a significant amount of space: + cam_ts_save: true + + #Overwrite baseline time series files, if found? + #If set to false, then time series creation will be skipped if files are found: + cam_overwrite_ts: false + + #Location where time series files are (or will be) stored: + cam_ts_loc: ${ts_loc}${diag_cam_baseline_climo.cam_case_name}/yrs_1995_2001/ + + +#This fourth set of variables provides settings for calling the Climate Variability +# Diagnostics Package (CVDP). If cvdp_run is set to true the CVDP will be set up and +# run in background mode, likely completing after the ADF has completed. +# If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed +# in the diag_var_list variable listing. +# For more CVDP information: https://www.cesm.ucar.edu/working_groups/CVC/cvdp/ +diag_cvdp_info: + + # Run the CVDP on the listed run(s)? + cvdp_run: false + + # CVDP code path, sets the location of the CVDP codebase + # CGD systems path = /home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ + # CISL systems path = /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ + # github location = https://github.com/NCAR/CVDP-ncl + cvdp_codebase_loc: /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ + + # Location where cvdp codebase will be copied to and diagnostic plots will be stored + cvdp_loc: ${diag_loc} + + # tar up CVDP results? + #cvdp_tar: false + + +#+++++++++++++++++++++++++++++++++++++++++++++++++++ +#These variables below only matter if you are using +#a non-standard method, or are adding your own +#diagnostic scripts. +#+++++++++++++++++++++++++++++++++++++++++++++++++++ + +#Note: If you want to pass arguments to a particular script, you can +#do it like so (using the "averaging_example" script in this case): +# - {averaging_example: {kwargs: {clobber: true}}} + +#Name of time-averaging scripts being used to generate climatologies. +#These scripts must be located in "scripts/averaging": +time_averaging_scripts: + - create_climo_files + #- create_TEM_files + +#Name of regridding scripts being used. +#These scripts must be located in "scripts/regridding": +regridding_scripts: + - regrid_and_vert_interp + +#List of analysis scripts being used. +#These scripts must be located in "scripts/analysis": +analysis_scripts: + - amwg_table + +#List of plotting scripts being used. +#These scripts must be located in "scripts/plotting": +plotting_scripts: + - global_latlon_map + - zonal_mean + - meridional_mean + - polar_map + - global_latlon_vect_map + - cam_taylor_diagram + - qbo + #- seasonal_cycle + #- tem + - tape_recorder + +#List of CAM variables that will be processesd: +#If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed +diag_var_list: + - AODDUSTdn + - AODVISdn + - CLDHGH + - CLDICE + - CLDLIQ + - CLDLOW + - CLDMED + - CLDTOT + - CLOUD + - FLNS + - FLNT + - FLNTC + - FSNS + - FSNT + - FSNTC + - LHFLX + - LWCF + - OMEGA500 + - PBLH + - PRECL + - PRECT + - PRECSL + - PRECSC + - PRECC + - PS + - PSL + - QFLX + - Q + - RELHUM + - SHFLX + - SST + - SWCF + - T + - TAUX + - TAUY + - TGCLDIWP + - TGCLDLWP + - TMQ + - TREFHT + - TS + - U + - U10 + - ICEFRAC + - OCNFRAC + - LANDFRAC + - RESTOM + - BC + - POM + - SO4 + - SOA + - NH4HSO4 + - DUST + - SeaSalt + + + + +# + +# Options for TEM diagnostics (./averaging/create_TEM_files.py and ./plotting/temp.py) +tem_info: + #Location where TEM files are stored: + #If path not specified or commented out, skip TEM calculation + #If no path, diagnostics wont run even if declared in averaging/plotting scripts below + tem_loc: /glade/scratch/richling/adf-output/ADF-data/TEM/ + + #TEM history file number + hist_num: h4 + + overwrite_tem_case: false + overwrite_tem_base: false + +#END OF FILE diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index 0b64311a5..ad0a3841d 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -2390,7 +2390,7 @@ utendwtem: budget_tables: # INPUTS #list of the gaseous variables to be caculated. - GAS_VARIABLES: ['CH4','CH3CCL3', 'CO', 'O3', 'ISOP', 'MTERP', 'CH3OH', 'CH3COCH3'] + GAS_VARIABLES: ['CH4','CH3CCL3', 'CO', 'O3', 'ISOP', 'MTERP', 'CH3OH', 'CH3COCH3','DMS','DMS_OASISS'] # list of the aerosol variables to be caculated. AEROSOL_VARIABLES: ['AOD','SOA', 'SALT', 'DUST', 'POM', 'BC', 'SO4'] @@ -2428,7 +2428,10 @@ budget_tables: 'DUST':168.0456, 'CH3CCL3':133.4042, 'CH3OH':32, - 'CH3COCH3':58} + 'CH3COCH3':58, + 'DMS':62.136, + 'DMS_OASISS':62.136, + 'AOD':1} # Avogadro's Number AVO: 6.022e23 diff --git a/lib/adf_web.py b/lib/adf_web.py index e39981f8f..f291b1b81 100644 --- a/lib/adf_web.py +++ b/lib/adf_web.py @@ -117,7 +117,7 @@ def __init__(self, config_file, debug=False): #Extract needed variables from yaml file: case_names = self.get_cam_info('cam_case_name', required=True) - + print(case_names) #Also extract baseline case (if applicable), and append to case_names list: if not self.compare_obs: baseline_name = self.get_baseline_info('cam_case_name', required=True) @@ -292,7 +292,9 @@ def add_website_data(self, web_data, web_name, case_name, if self.num_cases > 1: html_file = self.__case_web_paths['multi-case']["table_pages_dir"] / html_name else: - html_file = self.__case_web_paths[case_name]["table_pages_dir"] / html_name + #try: + html_file = self.__case_web_paths[case_name]["table_pages_dir"] / html_name + #else #End if asset_path = None else: @@ -787,4 +789,4 @@ def jinja_enumerate(arg): print(" ...Webpages have been generated successfully.") #++++++++++++++++++++ #End Class definition -#++++++++++++++++++++ \ No newline at end of file +#++++++++++++++++++++ diff --git a/scripts/analysis/aerosol_gas_tables.py b/scripts/analysis/aerosol_gas_tables.py index 32719bb1d..8f519adbf 100644 --- a/scripts/analysis/aerosol_gas_tables.py +++ b/scripts/analysis/aerosol_gas_tables.py @@ -1,25 +1,16 @@ import numpy as np import xarray as xr -import sys from pathlib import Path -import warnings # use to warn user about missing files. from datetime import datetime import numpy as np import itertools - -try: - import pandas as pd -except ImportError: - print("Pandas module does not exist in python path, but is needed for amwg_table.") - print("Please install module, e.g. 'pip install pandas'.") - sys.exit(1) -#End except +import pandas as pd # Import necessary ADF modules: from adf_base import AdfError -def aerosol_gas_tables(adfobj): +def aerosol_gas_tables(adfobj, trop_val=None, **kwargs): ''' Calculate aerosol and gaseous budget tables @@ -103,13 +94,25 @@ def aerosol_gas_tables(adfobj): * Add a condition to calculate whole world budgets when O3 is not find. * Update pressure calculation in a more general way. + Behrooz Roozitalab, 20 Aug, 2025 + - fixed: + * the html page was not created, it is fixed. + * added "hm" as a case to enable using annual averaged files in addition to monthly files. + * if config arg "trop_val" is pres500 : This version uses 500hPa as the tropopause threshold. + * if config arg "trop_val" is tropopause (realistic case) : New method of defining troposphere, use ozone (150ppb) or Trop_P. If not found, calculate total column + * Added DMS to gases list - reported as DMS not S + * Automatic addition of gaseous compounds even when not defined in the default list, + * based on Carbon MW (12). It still needs ADF modification to read a list from yaml file. ''' - #Notify user that script has started: msg = "\n Calculating chemistry/aerosol budget tables..." print(f"{msg}\n {'-' * (len(msg)-3)}") + # Check which type of tables to be created, default to 'troposphere' + if trop_val is None: + trop_val = 'tropopause' + # Inputs #------- # Variable defaults info @@ -132,7 +135,7 @@ def aerosol_gas_tables(adfobj): # ------------------- # if True, calculate only Tropospheric values # if False, all layers - # tropopause is defiend as o3>150ppb. If needed, change accordingly. + # tropopause is defiend as either directly or indirectly. Look for tropopause to see the definition Tropospheric = bres['Tropospheric'] ### NOT WORKING FOR NOW @@ -146,6 +149,13 @@ def aerosol_gas_tables(adfobj): # For SO4, we report everything in terms of Sulfur, so we use Sulfur MW here MW = bres['MW'] + # automatic generation of MW + for var in VARIABLES: + if var not in MW.keys(): + print(f"using Carbon molecular weight for {var}") + MW[var]=12 + + # Avogadro's Number AVO = float(bres['AVO']) # gravity @@ -203,7 +213,7 @@ def aerosol_gas_tables(adfobj): # Filter the list to include only strings that are possible h0 strings # - Search for either h0 or h0a - substrings = {"cam.h0","cam.h0a"} + substrings = {"cam.h0","cam.h0a","cam.hm"} case_hist_strs = [] for cam_case_str in hist_strs: # Check each possible h0 string @@ -248,8 +258,8 @@ def aerosol_gas_tables(adfobj): start_year = start_years[i] end_year = end_years[i] + 1 - start_date = f"{start_year}-1-1" - end_date = f"{end_year}-1-1" + start_date = f"{start_year:04d}-1-1" + end_date = f"{end_year:04d}-1-1" # Create time periods start_period = datetime.strptime(start_date, "%Y-%m-%d") @@ -273,7 +283,7 @@ def aerosol_gas_tables(adfobj): ListVars = list(tmp_file.variables) # Set up and fill dictionaries for components for current cases - dic_SE = set_dic_SE(ListVars,ext1_SE) + dic_SE = set_dic_SE(ListVars,ext1_SE,VARIABLES) dic_SE = fill_dic_SE(adfobj, dic_SE, VARIABLES, ListVars, ext1_SE, AEROSOLS, MW, AVO, gr, Mwair) text = f'\n\t Calculating values for {case}' @@ -282,7 +292,7 @@ def aerosol_gas_tables(adfobj): # Gather dictionary data for current case # NOTE: The calculations can take a long time... - Dic_crit, Dic_scn_var_comp[case],Tropospheric = make_Dic_scn_var_comp(adfobj, VARIABLES, data_dir, dic_SE, Files, ext1_SE, AEROSOLS,Tropospheric) + Dic_crit, Dic_scn_var_comp[case],Tropospheric,tropospheric_method = make_Dic_scn_var_comp(adfobj, VARIABLES, data_dir, dic_SE, Files, ext1_SE, AEROSOLS,Tropospheric,trop_val) # Regional refinement # NOTE: This function 'Inside_SE' is unavailable at the moment! - JR 10/2024 if regional: @@ -297,8 +307,17 @@ def aerosol_gas_tables(adfobj): # Set critical threshold current_crit = Dic_crit if Tropospheric: - trop = np.where(current_crit>150,np.nan,current_crit) - #strat=np.where(current_crit>150,current_crit,np.nan) + if tropospheric_method=='pressure': + # using pressure > 500hPa + trop = np.where(current_crit<500,np.nan,current_crit) + if tropospheric_method=='ozone': + # using ozone <150 ppb + trop = np.where(current_crit>150,np.nan,current_crit) + elif tropospheric_method=='tropopause': + # using pressure > tropopause pressure + trop = np.where(current_crit['Pressure']500 hPa" + print(msg) + except: + current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['U'],ext1_SE) + Dic_crit=current_crit['U'] + Tropospheric=False + msg += f"\n\t WARNING: No way of defining troposphere was found in the model, budgets are total column" + print(msg) + # Log info to logging file + msg = f"chem/aerosol tables:" + msg += f"\n\t - potential missing variables from budget? {missing_vars_tot}" + adfobj.debug_log(msg) + print(msg) + elif trop_val == 'tropopause': + try: + current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['O3'],ext1_SE) + Dic_crit=current_crit['O3'] + tropospheric_method='ozone' + msg += f"\n\t WARNING: Troposphere is defined as O3<150 ppb" + print(msg) + except: + try: + current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['TROP_P','Pressure'],ext1_SE) + Dic_crit=current_crit #[['TROP_P','Pressure']] + tropospheric_method='tropopause' + msg += f"\n\t WARNING: Troposphere is defined as pressure>trop_p" + print(msg) + except: + current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['U'],ext1_SE) + Dic_crit=current_crit['U'] + Tropospheric=False + msg += f"\n\t WARNING: No way of defining troposphere was found in the model, budgets are total column" + print(msg) msg = f"chem/aerosol tables:" msg += f"\n\t - needed variables for budget {needed_vars_tot}" adfobj.debug_log(msg) + print(msg) - return Dic_crit,Dic_scn_var_comp,Tropospheric + return Dic_crit,Dic_scn_var_comp,Tropospheric,tropospheric_method ##### @@ -923,7 +984,7 @@ def SEbudget(adfobj,dic_SE,data_dir,files,vars,ext1_SE,**kwargs): chunk of code that takes the longest by far. Example: - ** This is for both chemistry and aeorosl calculations + ** This is for both chemistry and aerosol calculations dic_SE: dictionary specyfing what variables to get. For example, for precipitation you can define SE as: @@ -960,11 +1021,14 @@ def SEbudget(adfobj,dic_SE,data_dir,files,vars,ext1_SE,**kwargs): if file==0: mock_2d=np.zeros_like(np.array(ds['PS'+ext1_SE].isel(time=0))) mock_3d=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) - + + if 'ncol' in list(ds.dims.keys()): + SE=True + else: + SE=False try: delP=np.array(ds['PDELDRY'+ext1_SE].isel(time=0)) except: - hyai=np.array(ds['hyai']) hybi=np.array(ds['hybi']) @@ -973,10 +1037,12 @@ def SEbudget(adfobj,dic_SE,data_dir,files,vars,ext1_SE,**kwargs): except: PS=np.array(ds['PS'+ext1_SE].isel(time=0)) # End try/except - P0=1e5 - Plevel=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) - + if SE: + Plevel=np.zeros((len(hyai),len(PS))) + else: + Plevel=np.zeros((len(hyai),len(PS),len(PS[0]))) + for i in range(len(Plevel)): Plevel[i]=hyai[i]*P0+hybi[i]*PS @@ -986,39 +1052,51 @@ def SEbudget(adfobj,dic_SE,data_dir,files,vars,ext1_SE,**kwargs): if file == 0: Dic_all_data[var]=[] - # Star gathering of variable data - data=[] - for i in dic_SE[var].keys(): - - if file == 0: - msg = f"chem/aerosol tables: 'SEbudget'" - msg += f"\n\t\t ** variable(s) needed for derived var {var}: {dic_SE[var].keys()}" - msg += f"\n\t\t - constituent for derived var {var}: {i}" - adfobj.debug_log(msg) - if i not in needed_vars: - needed_vars.append(i) - if ((i!='PS'+ext1_SE) and (i!='U'+ext1_SE) ) : - data.append(np.array(ds[i].isel(time=0))*dic_SE[var][i]) - else: - if i=='PS'+ext1_SE: - data.append(mock_2d) + if var=='TROP_P': + data=np.array(ds['TROP_P'+ext1_SE].isel(time=0))/100 + elif var== 'Pressure': + try: + data=np.array(ds['PMID'+ext1_SE].isel(time=0))/100 + except: + hyam=np.array(ds['hyam']) + hybm=np.array(ds['hybm']) + try: + PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) + except: + PS=np.array(ds['PS'+ext1_SE].isel(time=0)) + P0=1e5 + data=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) + for i in range(len(data)): + data[i]=hyam[i]*P0+hybm[i]*PS + data=data/100 + else: + data=[] + for i in dic_SE[var].keys(): + if file == 0: + msg = f"chem/aerosol tables: 'SEbudget'" + msg += f"\n\t\t ** variable(s) needed for derived var {var}: {dic_SE[var].keys()}" + msg += f"\n\t\t - constituent for derived var {var}: {i}" + adfobj.debug_log(msg) + if i not in needed_vars: + needed_vars.append(i) + if ((i!='PS'+ext1_SE) and (i!='U'+ext1_SE) ) : + data.append(np.array(ds[i].isel(time=0))*dic_SE[var][i]) else: - data.append(mock_3d) + if i=='PS'+ext1_SE: + data.append(mock_2d) + else: + data.append(mock_3d) + if file == 0: + if var not in missing_vars: + if var!='U': # This is to avoid confusion between U variable or U mock! + missing_vars.append(var) + msg += f"\n\t\t - no variable was found for var {var}: {i}" + # End if - if file == 0: - - if var not in missing_vars: - if var!='U': # This is to avoid confusion between U variable or U mock! - missing_vars.append(var) - msg += f"\n\t\t - no variable was found for var {var}: {i}" - - # End if - - # Get total summed data for this history file data - data=np.sum(data,axis=0) - - # End try/except + + # Get total summed data for this history file data + data=np.sum(data,axis=0) if ('CHML' in var) or ('CHMP' in var) : Temp=np.array(ds['T'+ext1_SE].isel(time=0)) @@ -1137,7 +1215,7 @@ def calc_budget_data(current_var, Dic_scn_var_comp, area, trop, inside, num_yrs, spc_wdf = spc_wdfa +spc_wdfc tmp_wdf = spc_wdf wdf = np.ma.masked_where(inside==False,tmp_wdf*area) #convert Kg/m2/s to Tg/yr - WDF = np.ma.sum(wdf*duration*1e-9)/num_yrs + WDF = -1*np.ma.sum(wdf*duration*1e-9)/num_yrs chem_dict[f"{current_var}_WETDEP (Tg{specifier}/yr)"] = np.round(WDF,5) if current_var in ["SOA",'SO4']: @@ -1272,6 +1350,9 @@ def make_table(adfobj, vars, chem_type, Dic_scn_var_comp, areas, trops, case_nam if val != 0: # Skip variables with a value of 0 print(f"\t - Variable '{key}' being added to table") rows.append({'variable': key, nickname: np.round(val, 3)}) + elif 'OASISS_EMIS (' in key: # the paranthesis is to ignore EMIS_Elevated variables! + print(f"\t - Variable '{key}' being added to table") + rows.append({'variable': key, nickname: np.round(val, 3)}) else: msg = f"chem/aerosol tables:" msg += f"\n\t - Variable '{key}' has value of 0, will not add to table" @@ -1311,5 +1392,7 @@ def make_table(adfobj, vars, chem_type, Dic_scn_var_comp, areas, trops, case_nam output_csv_file = output_location / f'ADF_amwg_{chem_type}_table.csv' # Save table to CSV and add table dataframe to website (if enabled) table_df.to_csv(output_csv_file, index=False) - adfobj.add_website_data(table_df, chem_type, case, plot_type="Tables") + #adfobj.add_website_data(table_df, chem_type, case, plot_type="Tables") + adfobj.add_website_data(table_df, chem_type, case_names[0], plot_type="Tables") + ##### diff --git a/scripts/analysis/aerosol_gas_tables_Tropopause_version0.py b/scripts/analysis/aerosol_gas_tables_Tropopause_version0.py new file mode 100644 index 000000000..e8759678c --- /dev/null +++ b/scripts/analysis/aerosol_gas_tables_Tropopause_version0.py @@ -0,0 +1,1390 @@ +import numpy as np +import xarray as xr +import sys +from pathlib import Path +import warnings # use to warn user about missing files. + +from datetime import datetime +import numpy as np +import itertools + +try: + import pandas as pd +except ImportError: + print("Pandas module does not exist in python path, but is needed for amwg_table.") + print("Please install module, e.g. 'pip install pandas'.") + sys.exit(1) +#End except + +# Import necessary ADF modules: +from adf_base import AdfError + +def aerosol_gas_tables(adfobj): + ''' + Calculate aerosol and gaseous budget tables + + Default set of variables: change in lib/adf_variable_defaults.yaml + ------------------------- + GAS_VARIABLES: ['CH4','CH3CCL3', 'CO', 'O3', 'ISOP', 'MTERP', 'CH3OH', 'CH3COCH3'] + AEROSOL_VARIABLES: ['AOD','SOA', 'SALT', 'DUST', 'POM', 'BC', 'SO4'] + + Default output for tables: + + Gases: + ------ + CH4_BURDEN (Tg), CH4_CHEM_LOSS (Tg/yr), CH4_LIFETIME (years) + + CH3CCL3_BURDEN (Tg), CH3CCL3_CHEM_LOSS (Tg/yr), CH3CCL3_LIFETIME (days) + + CO_EMIS (Tg/yr), CO_BURDEN (Tg), CO_CHEM_LOSS (Tg/yr), CO_CHEM_PROD (Tg/yr), CO_DRYDEP (Tg/yr) + CO_TDEP (Tg/yr), CO_LIFETIME (days), CO_TEND (Tg/yr) + + O3_BURDEN (Tg), O3_CHEM_LOSS (Tg/yr), O3_CHEM_PROD (Tg/yr), O3_DRYDEP (Tg/yr), O3_TDEP (Tg/yr) + O3_LIFETIME (days), O3_TEND (Tg/yr), O3_STE (Tg/yr) + + LNOx_PROD (Tg N/yr) + + ISOP_EMIS (Tg/yr), ISOP_BURDEN (Tg) + + Monoterpene_EMIS (Tg/yr), Monoterpene_BURDEN (Tg) + + Methanol_EMIS (Tg/yr), Methanol_BURDEN (Tg), Methanol_DRYDEP (Tg/yr), Methanol_WETDEP (Tg/yr), Methanol_TDEP (Tg/yr) + + Acetone_EMIS (Tg/yr), Acetone_BURDEN (Tg), Acetone_DRYDEP (Tg/yr), Acetone_WETDEP (Tg/yr), Acetone_TDEP (Tg/yr) + + + + Aerosols: + --------- + AOD_mean + + SOA_BURDEN (Tg), SOA_CHEM_LOSS (Tg/yr), SOA_DRYDEP (Tg/yr), SOA_WETDEP (Tg/yr), SOA_GAEX (Tg/yr), SOA_LIFETIME (days) + + SALT_EMIS (Tg/yr), SALT_BURDEN (Tg), SALT_DRYDEP (Tg/yr), SALT_WETDEP (Tg/yr), SALT_LIFETIME (days) + + DUST_EMIS (Tg/yr), DUST_BURDEN (Tg), DUST_DRYDEP (Tg/yr), DUST_WETDEP (Tg/yr), DUST_LIFETIME (days) + + POM_EMIS (Tg/yr), POM_BURDEN (Tg), POM_DRYDEP (Tg/yr), POM_WETDEP (Tg/yr), POM_LIFETIME (days) + + BC_EMIS (Tg/yr), BC_BURDEN (Tg), BC_DRYDEP (Tg/yr), BC_WETDEP (Tg/yr), BC_LIFETIME (days) + + SO4_EMIS_elevated (Tg S/yr), SO4_BURDEN (Tg S), SO4_DRYDEP (Tg S/yr), SO4_WETDEP (Tg S/yr), SO4_GAEX (Tg S/yr) + SO4_LIFETIME (days), SO4_AQUEOUS (Tg S/yr), SO4_NUCLEATION (Tg S/yr) + + + List of variable names and descriptions for clarity + --------------------------------------------------- + - ListVars: list of all available variables from given history file + - GAS_VARIABLES: list fo necessary CAM gaseous variables + - AEROSOL_VARIABLES: list fo necessary CAM aerosol variables + - AEROSOLS: list of necessary aerosols for computations + + + MODIFICATION HISTORY: + Behrooz Roozitalab, 02, NOV, 2022: VERSION 1.00 + - Initial version + + Justin Richling, 27 Nov, 2023 + - updated to fit to ADF and check with old AMWG chem/aerosol tables + - fixed: + * added difference bewtween cases column to tables + + Behrooz Roozitalab, 8 Aug, 2024 + - fixed: + * lifetime inconsitencies + * Removed redundant calculations to improve the speed + * Verified the results against the NCL script. + + Behrooz Roozitalab, 5 Jun, 2025 + - fixed: + * Fix the bugs in the calculation (when converting from Jupyterhub to ADF) + * add the 'U' variable in dic_SE + * make the code faster by modifying make_Dic_scn_var_comp + * Add a condition to calculate whole world budgets when O3 is not find. + * Update pressure calculation in a more general way. + + Behrooz Roozitalab, 20 Aug, 2025 _ Version 0 + - fixed: + * the html page was not created, it is fixed. + * added "hm" as a case to enable using annual averaged files in addition to monthly files. + * New method of defining troposphere, use ozone (150ppb) or Trop_P. If not found, calculate total column + * Added DMS to gases list - reported as DMS not S + * Automatic addition of gaseous compounds even when not defined in the default list, + * based on Carbon MW (12). It still needs ADF modification to read a list from yaml file. + ''' + + + #Notify user that script has started: + msg = "\n Calculating chemistry/aerosol budget tables..." + print(f"{msg}\n {'-' * (len(msg)-3)}") + + # Inputs + #------- + # Variable defaults info + res = adfobj.variable_defaults # dict of variable-specific plot preferences + bres = res['budget_tables'] + # list of the gaseous variables to be caculated. + GAS_VARIABLES = bres['GAS_VARIABLES'] + + # list of the aerosol variables to be caculated. + AEROSOL_VARIABLES = bres['AEROSOL_VARIABLES'] + + #list of all the variables to be caculated. + VARIABLES = GAS_VARIABLES + AEROSOL_VARIABLES + + # For the case that outputs are saved for a specific region. + # i.e., when using fincllonlat in user_nl_cam + ext1_SE = bres['ext1_SE'] + + # Tropospheric Values + # ------------------- + # if True, calculate only Tropospheric values + # if False, all layers + # tropopause is defiend as either directly or indirectly. Look for tropopause to see the definition + Tropospheric = bres['Tropospheric'] + + ### NOT WORKING FOR NOW + # To calculate the budgets only for a region + # Lat/Lon extent + limit = bres['limit'] + regional = bres['regional'] + + # Dictionary for Molecular weights. Keys must be consistent with variable name + # For aerosols, the MW is used only for chemical loss, chemical production, and elevated emission calculations + # For SO4, we report everything in terms of Sulfur, so we use Sulfur MW here + MW = bres['MW'] + + # automatic generation of MW + for var in VARIABLES: + if var not in MW.keys(): + print(f"using Carbon molecular weight for {var}") + MW[var]=12 + + + # Avogadro's Number + AVO = float(bres['AVO']) + # gravity + gr = float(bres['gr']) + # Mw air + Mwair = float(bres['Mwair']) + + # The variables in the list below must be aerosols - do not add AOD and DAOD + # no need to change this list, unless for a specific need! + AEROSOLS = bres['AEROSOLS'] + + # Start gathering case, path, and data info + #----------------------------------------- + + # CAM simulation variables (these quantities are always lists): + case_names = adfobj.get_cam_info('cam_case_name', required=True) + + # Grab all case nickname(s) + test_nicknames_list = adfobj.case_nicknames["test_nicknames"] + nicknames_list = test_nicknames_list + # Grab climo years + start_years = adfobj.climo_yrs["syears"] + end_years = adfobj.climo_yrs["eyears"] + + #Grab history strings: + hist_strs = adfobj.hist_string["test_hist_str"] + + # Grab history file locations from config yaml file + hist_locs = adfobj.get_cam_info("cam_hist_loc", required=True) + + # Check if this is test model vs baseline model + # If so, update test case(s) lists created above + if not adfobj.compare_obs: + # Get baseline case info + case_names += [adfobj.get_baseline_info("cam_case_name")] + nicknames_list += [adfobj.case_nicknames["base_nickname"]] + + # Grab climo years + start_years += [adfobj.climo_yrs["syear_baseline"]] + end_years += [adfobj.climo_yrs["eyear_baseline"]] + + # Get history file info + hist_strs += [adfobj.hist_string["base_hist_str"]] + hist_locs += [adfobj.get_baseline_info("cam_hist_loc")] + # End if + + # Check to ensure number of case names matches number history file locations. + # If not, exit script + if len(hist_locs) != len(case_names): + errmsg = "Error: number of cases does not match number of history file locations. Script is exiting." + raise AdfError(errmsg) + + # Initialize nicknames dictionary + #nicknames = {} + + # Filter the list to include only strings that are possible h0 strings + # - Search for either h0 or h0a + substrings = {"cam.h0","cam.h0a","cam.hm"} + case_hist_strs = [] + for cam_case_str in hist_strs: + # Check each possible h0 string + for string in cam_case_str: + if string in substrings: + case_hist_strs.append(string) + break + + # Create path object for the CAM history file(s) location: + data_dirs = [] + for case_idx,case in enumerate(nicknames_list): + + print(f"\t Looking for history location: {hist_locs[case_idx]}") + + + #Check that history file input directory actually exists: + if (hist_locs[case_idx] is None) or (not Path(hist_locs[case_idx]).is_dir()): + errmsg = f"History files directory '{hist_locs[case_idx]}' not found. Script is exiting." + raise AdfError(errmsg) + + #Write to debug log if enabled: + adfobj.debug_log(f"DEBUG: location of history files is {str(hist_locs[case_idx])}") + # Update list for found directories + data_dirs.append(hist_locs[case_idx]) + + # End gathering case, path, and data info + #----------------------------------------- + # Periods of Interest + # ------------------- + # choose the period of interest. Plots will be averaged within this period + durations = {} + num_yrs = {} + + # Main function + #-------------- + # Set dictionary of components for each case + Dic_scn_var_comp = {} + areas = {} + trops = {} + insides = {} + for i,case in enumerate(nicknames_list): + + start_year = start_years[i] + end_year = end_years[i] + 1 + start_date = f"{start_year}-1-1" + end_date = f"{end_year}-1-1" + + # Create time periods + start_period = datetime.strptime(start_date, "%Y-%m-%d") + end_period = datetime.strptime(end_date, "%Y-%m-%d") + + # Calculated duration of time period in seconds? + durations[case] = (end_period-start_period).days*86400 #+365*86400 + + + # Get number of years for calculations + num_yrs[case] = (int(end_year)-int(start_year)) #+1 + + # Get currenty history file directory + data_dir = data_dirs[i] + + # Get all files, lats, lons, and area weights for current case + Files,Lats,Lons,areas[case],ext1_SE = Get_files(adfobj,data_dir,start_year,end_year,case_hist_strs[i],area=True) + # find the name of all the variables in the file. + # this will help the code to work for the variables that are not in the files (assingn 0s) + tmp_file = xr.open_dataset(Path(data_dir) / Files[0]) + ListVars = list(tmp_file.variables) + + # Set up and fill dictionaries for components for current cases + dic_SE = set_dic_SE(ListVars,ext1_SE,VARIABLES) + dic_SE = fill_dic_SE(adfobj, dic_SE, VARIABLES, ListVars, ext1_SE, AEROSOLS, MW, AVO, gr, Mwair) + + text = f'\n\t Calculating values for {case}' + print(text) + print("\t " + "-" * (len(text) - 2)) + + # Gather dictionary data for current case + # NOTE: The calculations can take a long time... + Dic_crit, Dic_scn_var_comp[case],Tropospheric,tropospheric_method = make_Dic_scn_var_comp(adfobj, VARIABLES, data_dir, dic_SE, Files, ext1_SE, AEROSOLS,Tropospheric) + # Regional refinement + # NOTE: This function 'Inside_SE' is unavailable at the moment! - JR 10/2024 + if regional: + #inside = Inside_SE_region(current_lat,current_lon,dir_shapefile) + inside = Inside_SE(Lats,Lons,limit) + else: + if len(np.shape(areas[case])) == 1: + inside = np.full((len(Lons)),True) + else: + inside = np.full((len(Lats),len(Lons)),True) + + # Set critical threshold + current_crit = Dic_crit + if Tropospheric: + if tropospheric_method=='ozone': + # using ozone <150 ppb + trop = np.where(current_crit>150,np.nan,current_crit) + elif tropospheric_method=='tropopause': + # using pressure > tropopause pressure + trop = np.where(current_crit['Pressure']150,current_crit,np.nan) + else: + trop=current_crit + trops[case] = trop + insides[case] = inside + + # Make and save tables + table_kwargs = {"adfobj":adfobj, + "Dic_scn_var_comp":Dic_scn_var_comp, + "areas":areas, + "trops":trops, + "case_names":case_names, + "nicknames":nicknames_list, + "durations":durations, + "insides":insides, + "num_yrs":num_yrs, + "AEROSOLS":AEROSOLS} + + #print(table_kwargs) + + # Create the budget tables + #------------------------- + # Aerosols + if len(AEROSOL_VARIABLES) > 0: + print("\tMaking table for aerosols") + make_table(vars=AEROSOL_VARIABLES, chem_type='aerosols', **table_kwargs) + # Gases + if len(GAS_VARIABLES) > 0: + print("\tMaking table for gases") + make_table(vars=GAS_VARIABLES, chem_type='gases', **table_kwargs) +####### + +################## +# Helper functions +################## + +def list_files(adfobj, directory, start_year ,end_year, h_case): + + """ + This function extracts the files in the directory that are within the chosen dates + and history number. + """ + + # History file year range + yrs = np.arange(int(start_year), int(end_year)) + + all_filenames = [] + for i in yrs: +# all_filenames.append(sorted(Path(directory).glob(f'*.{h_case}.{i}-*'))) + all_filenames.append(sorted(Path(directory).glob(f'*.{h_case}.{i}*'))) + + #print(directory) + # Flattening the list of lists + filenames = list(itertools.chain.from_iterable(sorted(all_filenames))) + if len(filenames)==0: + #sys.exit(" Directory has no outputs ") + msg = f"chem/aerosol tables, 'list_files':" + msg += f"\n\t - Directory '{directory}' has no outputs." + adfobj.debug_log(msg) + + return filenames +##### + + +def Get_files(adfobj, data_dir, start_year, end_year, h_case, **kwargs): + + """ + This function retrieves the files, latitude, and longitude information + in all the directories within the chosen dates. + """ + ext1_SE = kwargs.pop('ext1_SE','') + area = kwargs.pop('area',False) + + Earth_rad=6.371e6 # Earth Radius in meters + + current_files = list_files(adfobj, data_dir, start_year, end_year,h_case) + # get the Lat and Lons for each case + tmp_file = xr.open_dataset(Path(data_dir) / current_files[0]) + lon = tmp_file['lon'+ext1_SE].data + lon[lon > 180.] -= 360 # shift longitude from 0-360˚ to -180-180˚ + lat = tmp_file['lat'+ext1_SE].data + + if area == True: + try: + tmp_area = tmp_file['area'+ext1_SE].data + Earth_area = 4 * np.pi * Earth_rad**(2) + + areas = tmp_area*Earth_area/np.nansum(tmp_area) + except KeyError: + try: + tmp_area = tmp_file['AREA'+ext1_SE].isel(time=0).data + areas=tmp_area + #Earth_area = 4 * np.pi * Earth_rad**(2) + #areas = tmp_area*Earth_area/np.nansum(tmp_area) + except: + dlon = np.abs(lon[1]-lon[0]) + dlat = np.abs(lat[1]-lat[0]) + + lon2d,lat2d = np.meshgrid(lon,lat) + #area=np.zeros_like(lat2d) + + dy = Earth_rad*dlat*np.pi/180 + dx = Earth_rad*np.cos(lat2d*np.pi/180)*dlon*np.pi/180 + + tmp_area = dx*dy + areas = tmp_area + # End if + + # Variables to return + return current_files,lat,lon,areas,ext1_SE +##### + +def set_dic_SE(ListVars, ext1_SE,variables): + """ + Initialize dictionary to house all the relevant tabel data + """ + + # Initialize dictionary + #---------------------- + dic_SE={} + + # Chemistry + #---------- + dic_SE['U']={'U'+ext1_SE:1} + dic_SE['O3']={'O3'+ext1_SE:1e9} # covert to ppb for Tropopause calculation + dic_SE['CH4']={'CH4'+ext1_SE:1} + dic_SE['CO']={'CO'+ext1_SE:1} + + dic_SE['ISOP']={'ISOP'+ext1_SE:1} + dic_SE['MTERP']={'MTERP'+ext1_SE:1} + dic_SE['CH3OH']={'CH3OH'+ext1_SE:1} + dic_SE['CH3COCH3']={'CH3COCH3'+ext1_SE:1} + dic_SE['CH3CCL3']={'CH3CCL3'+ext1_SE:1} + dic_SE['CHBR3']={'CHBR3'+ext1_SE:1} + dic_SE['CH2BR2']={'CH2BR2'+ext1_SE:1} + + # Aerosols + #--------- + + dic_SE['DAOD']={'AODDUSTdn'+ext1_SE:1} + dic_SE['AOD']={'AODVISdn'+ext1_SE:1} + + dic_SE['DUST']={'dst_a1'+ext1_SE:1, + 'dst_a2'+ext1_SE:1, + 'dst_a3'+ext1_SE:1} + + dic_SE['SALT']={'ncl_a1'+ext1_SE:1, + 'ncl_a2'+ext1_SE:1, + 'ncl_a3'+ext1_SE:1} + + dic_SE['POM']={'pom_a1'+ext1_SE:1, + 'pom_a4'+ext1_SE:1} + + dic_SE['BC']={'bc_a1'+ext1_SE:1, + 'bc_a4'+ext1_SE:1} + + + dic_SE['SO4']={'so4_a1'+ext1_SE:1, + 'so4_a2'+ext1_SE:1, + 'so4_a3'+ext1_SE:1, + 'so4_a5'+ext1_SE:1} + + # FOR SOA, first check if the integrated bins are included + if (('soa_a1'+ext1_SE in ListVars ) & ('soa_a1'+ext1_SE in ListVars )): + dic_SE['SOA'] = {'soa_a1'+ext1_SE:1, + 'soa_a2'+ext1_SE:1} + else: + dic_SE['SOA'] = {'soa1_a1'+ext1_SE:1, + 'soa2_a1'+ext1_SE:1, + 'soa3_a1'+ext1_SE:1, + 'soa4_a1'+ext1_SE:1, + 'soa5_a1'+ext1_SE:1, + 'soa1_a2'+ext1_SE:1, + 'soa2_a2'+ext1_SE:1, + 'soa3_a2'+ext1_SE:1, + 'soa4_a2'+ext1_SE:1, + 'soa5_a2'+ext1_SE:1} + + dic_SE['DMS']={'DMS'+ext1_SE:1} + #dic_SE['TROP_P']={'TROP_P'+ext1_SE:1} + + + # automatic generation of dic_SE + for var in variables: + if var not in dic_SE.keys(): + dic_SE[var]={var+ext1_SE:1} + + # consider for OASISS DMS separately + if var=='DMS': + dic_SE['DMS_OASISS']={'DMS_OASISS'+ext1_SE:1} + # End if + + return dic_SE +##### + +def fill_dic_SE(adfobj, dic_SE, variables, ListVars, ext1_SE, AEROSOLS, MW, AVO, gr, Mwair): + """ + Function for dealing with conversion factors for different components and filling the main data + dictionary 'dic_SE' + + Input dictionary and return updated dictionary 'dic_SE' + + Arguments + --------- + variables : list + - list of main variables? + ListVars : list + - list of ??????? + + Returns + ------- + dic_SE : dict + - full dictionary of derived variables + + Some conversion factors need density or Layer's pressure, that will be accounted for when reading the files. + We convert everying to kg/m2/s or kg/m2 or kg/s, so that final Tg/yr or Tg results are consistent + """ + + # Logging info message + msg = f"chem/aerosol tables: 'fill_dic_SE'" + + for var in variables: + + if 'AOD' in var: + dic_SE[var+'_AOD']={} + else: + dic_SE[var+'_BURDEN']={} + dic_SE[var+'_CHML']={} + dic_SE[var+'_CHMP']={} + + dic_SE[var+'_SF']={} + dic_SE[var+'_CLXF']={} + + dic_SE[var+'_DDF']={} + dic_SE[var+'_WDF']={} + + if var in AEROSOLS: + dic_SE[var+'_GAEX']={} + dic_SE[var+'_DDFC']={} + dic_SE[var+'_WDFC']={} + else: + dic_SE[var+'_TEND']={} + dic_SE[var+'_LNO']={} + # End if + + # We have nucleation and aqueous chemistry for sulfate. + if var=='SO4': + dic_SE[var+'_NUCL']={} + dic_SE[var+'_AQS']={} + # End if + + # Grab the variable keys + var_keys = dic_SE[var].keys() + + for key in var_keys: + msg += f"\n\t Creating component of {var}: {key}" + + # for CHML and CHMP: + # original unit : [molec/cm3/s] + # following Tilmes code to convert to [kg/m2/s] + # conversion: Mw*rho*delP*1e3/Avo/gr + # rho and delP will be applied when reading the files in SEbudget function. + + # for AOD and DAOD: + if 'AOD' in var: + if key in ListVars: + dic_SE[var+'_AOD'][key+ext1_SE]=1 + else: + dic_SE[var+'_AOD']['PS'+ext1_SE]=0. + # End if + continue # AOD doesn't need any other budget calculations + # End if + + # for CHML and CHMP: + # original unit : [molec/cm3/s] + # following Tilmes code to convert to [kg/m2/s] + # conversion: Mw*rho*delP*1e3/Avo/gr + # rho and delP will be applied when reading the files in SEbudget function. + if key=='O3'+ext1_SE: + # for O3, we should not include fast cycling reactions + # As a result, we use below diagnostics in the model if available. If not, we use CHML and CHMP + if ((key+'_Loss' in ListVars) & (key+'_Prod' in ListVars)) : + dic_SE[var+'_CHML'][key+'_Loss'+ext1_SE]=MW[var]*1e3/AVO/gr + dic_SE[var+'_CHMP'][key+'_Prod'+ext1_SE]=MW[var]*1e3/AVO/gr + else: + if key+'_CHML' in ListVars: + dic_SE[var+'_CHML'][key+'_CHML'+ext1_SE]=MW[var]*1e3/AVO/gr + else: + dic_SE[var+'_CHML']['U'+ext1_SE]=0 + # End if + + if key+'_CHMP' in ListVars: + dic_SE[var+'_CHMP'][key+'_CHMP'+ext1_SE]=MW[var]*1e3/AVO/gr + else: + dic_SE[var+'_CHMP']['U'+ext1_SE]=0 + # End if + # End if + else: + if key+'_CHML' in ListVars: + dic_SE[var+'_CHML'][key+'_CHML'+ext1_SE]=MW[var]*1e3/AVO/gr + else: + dic_SE[var+'_CHML']['U'+ext1_SE]=0 + # End if + + if key+'_CHMP' in ListVars: + dic_SE[var+'_CHMP'][key+'_CHMP'+ext1_SE]=MW[var]*1e3/AVO/gr + else: + dic_SE[var+'_CHMP']['U'+ext1_SE]=0 + # End if + # End if + + + # for SF: + # original unit: [kg/m2/s] + if 'SF'+key in ListVars: + if var=='SO4': + dic_SE[var+'_SF']['SF'+key+ext1_SE]=32.066/115.11 + else: + dic_SE[var+'_SF']['SF'+key+ext1_SE]=1 + elif ((var=='DMS_OASISS') & ('OCN_FLUX_DMS' in ListVars)): + dic_SE[var+'_SF']['OCN_FLUX_DMS'+ext1_SE]=1 + + # End if + elif key+'SF' in ListVars: + dic_SE[var+'_SF'][key+ext1_SE+'SF']=1 + else: + dic_SE[var+'_SF']['PS'+ext1_SE]=0. + # End if + + + # for CLXF: + # original unit: [molec/cm2/s] + # conversion: Mw*10/Avo + if key+'_CLXF' in ListVars: + dic_SE[var+'_CLXF'][key+'_CLXF'+ext1_SE]=MW[var]*10/AVO # convert [molec/cm2/s] to [kg/m2/s] + else: + dic_SE[var+'_CLXF']['PS'+ext1_SE]=0. + # End if + + # Aerosols + if var in AEROSOLS: + # for each species: + # original unit : [kg/kg] in dry air + # convert to [kg/m2] + # conversion: delP/gr + # delP will be applied when reading the files in SEbudget function. + if key in ListVars: + if var=='SO4': # For SO4, we report all the budget calculation for Sulfur. + dic_SE[var+'_BURDEN'][key+ext1_SE]=(32.066/115.11)/gr + else: + dic_SE[var+'_BURDEN'][key+ext1_SE]=1/gr + # End if + else: + dic_SE[var+'_BURDEN']['U'+ext1_SE]=0 + # End if + + + # for DDF: + # original unit: [kg/m2/s] + if key+'DDF' in ListVars: + if var=='SO4': + dic_SE[var+'_DDF'][key+ext1_SE+'DDF']=32.066/115.11 + else: + dic_SE[var+'_DDF'][key+ext1_SE+'DDF']=1 + # End if + else: + dic_SE[var+'_DDF']['PS'+ext1_SE]=0. + # End if + + + # for SFWET: + # original unit: [kg/m2/s] + if key+'SFWET' in ListVars: + if var=='SO4': + dic_SE[var+'_WDF'][key+ext1_SE+'SFWET']=32.066/115.11 + else: + dic_SE[var+'_WDF'][key+ext1_SE+'SFWET']=1 + # End if + else: + dic_SE[var+'_WDF']['PS'+ext1_SE]=0. + # End if + + + # for sfgaex1: + # original unit: [kg/m2/s] + if key+'_sfgaex1' in ListVars: + if var=='SO4': + dic_SE[var+'_GAEX'][key+ext1_SE+'_sfgaex1']=32.066/115.11 + else: + dic_SE[var+'_GAEX'][key+ext1_SE+'_sfgaex1']=1 + # End if + else: + dic_SE[var+'_GAEX']['PS'+ext1_SE]=0. + # End if + + + # for DDF in cloud water: + # original unit: [kg/m2/s] + cloud_key=key[:-2]+'c'+key[-1] + if cloud_key+ext1_SE+'DDF' in ListVars: + if var=='SO4': + dic_SE[var+'_DDFC'][cloud_key+ext1_SE+'DDF']=32.066/115.11 + else: + dic_SE[var+'_DDFC'][cloud_key+ext1_SE+'DDF']=1 + # End if + else: + dic_SE[var+'_DDFC']['PS'+ext1_SE]=0. + # End if + + # for SFWET in cloud water: + # original unit: [kg/m2/s] + if cloud_key+ext1_SE+'SFWET' in ListVars: + if var=='SO4': + dic_SE[var+'_WDFC'][cloud_key+ext1_SE+'SFWET']=32.066/115.11 + else: + dic_SE[var+'_WDFC'][cloud_key+ext1_SE+'SFWET']=1 + # End if + else: + dic_SE[var+'_WDFC']['PS'+ext1_SE]=0. + # End if + + if var=='SO4': + # for Nucleation : + # original unit: [kg/m2/s] + if key+ext1_SE+'_sfnnuc1' in ListVars: + dic_SE[var+'_NUCL'][key+ext1_SE+'_sfnnuc1']=32.066/115.11 + else: + dic_SE[var+'_NUCL']['PS'+ext1_SE]=0. + # End if + + # for Aqueous phase : + # original unit: [kg/m2/s] + if (('AQSO4_H2O2'+ext1_SE in ListVars) & ('AQSO4_O3'+ext1_SE in ListVars)) : + dic_SE[var+'_AQS']['AQSO4_H2O2'+ext1_SE]=32.066/115.11 + dic_SE[var+'_AQS']['AQSO4_O3'+ext1_SE]=32.066/115.11 + else: + # original unit: [kg/m2/s] + if cloud_key+'AQSO4'+ext1_SE in ListVars: + dic_SE[var+'_AQS'][cloud_key+'AQSO4'+ext1_SE]=32.066/115.11 + else: + dic_SE[var+'_AQS']['PS'+ext1_SE]=0. + # End if + + if cloud_key+'AQH2SO4'+ext1_SE in ListVars: + dic_SE[var+'_AQS'][cloud_key+'AQH2SO4'+ext1_SE]=32.066/115.11 + else: + dic_SE[var+'_AQS']['PS'+ext1_SE]=0. + # End if + # End if + # End if + + else: # Gases + # for each species: + # original unit : [mole/mole] in dry air + # convert to [kg/m2] + # conversion: Mw*delP/Mwair/gr Mwair=28.97 gr/mole + # delP will be applied when reading the files in SEbudget function. + if key in ListVars: + dic_SE[var+'_BURDEN'][key+ext1_SE]=MW[var]/Mwair/gr + else: + dic_SE[var+'_BURDEN']['U'+ext1_SE]=0 + # End if + + # for DF: + # original unit: [kg/m2/s] + if 'DF_'+key in ListVars: + dic_SE[var+'_DDF']['DF_'+key+ext1_SE]=1 + else: + dic_SE[var+'_DDF']['PS'+ext1_SE]=0. + # End if + + # for WD: + # original unit: [kg/m2/s] + if 'WD_'+key in ListVars: + dic_SE[var+'_WDF']['WD_'+key+ext1_SE]=1 + else: + dic_SE[var+'_WDF']['PS'+ext1_SE]=0. + # End if + + # for Chem tendency: + # original unit: [kg/s] + # conversion: not needed + if 'D'+key+'CHM' in ListVars: + dic_SE[var+'_TEND']['D'+key+'CHM'+ext1_SE]=1 # convert [kg/s] to [kg/s] + else: + dic_SE[var+'_TEND']['U'+ext1_SE]=0 + # End if + + # for Lightning NO production: (always in gas) + # original unit: [Tg N/Yr] + # conversion: not needed + if 'LNO_COL_PROD' in ListVars: + dic_SE[var+'_LNO']['LNO_COL_PROD'+ext1_SE]=1 # convert [Tg N/yr] to [Tg N /yr] + else: + dic_SE[var+'_LNO']['PS'+ext1_SE]=0 + # End if + # End if (aerosols or gases) + # End for + # End for + + # Write to log + adfobj.debug_log(msg) + + return dic_SE +##### + + +def make_Dic_scn_var_comp(adfobj, variables, current_dir, dic_SE, current_files, ext1_SE, AEROSOLS,Tropospheric): + """ + This function retrieves the files, latitude, and longitude information + in all the directories within the chosen dates. + + current_dir: list + - showing the directories to look for files. always end with '/' + + current_files: list + - List of CAM history files + + start_year: string + - Starting year + + end_year: string + - Ending year + + kwargs + ------ + ext1_SE: string + - specify if the files are for only a region, which changes to variable names. + ex: if you saved files for a only a box region ($LL_lat$,$LL_lon$,$UR_lat$,$UR_lon$), + the 'lat' variable will be saved as: 'lat_$LL_lon$e_to_$UR_lon$e_$LL_lat$n_to_$UR_lat$n' + for instance: 'lat_65e_to_91e_20n_to_32n' + + Returns + ------- + Dic_crit: + - dictionary for critical values for current case + Dic_scn_var_comp: + - full dictionary of all variables and components for current case + + NOTE: The LNO is lightning NOx, which should be reported explicitly rather as CO_LNO, O3_LNO, ... + """ + + # Set lists to gather necessary variables for logging + missing_vars_tot = [] + needed_vars_tot = [] + + # Initialize final component dictionary + Dic_var_comp={} + + for current_var in variables: + if 'AOD' in current_var: + components=[current_var+'_AOD'] + else: + if current_var in AEROSOLS: # AEROSOLS + + # Components are: burden, chemical loss, chemical prod, dry deposition, + # surface emissions, elevated emissions, wet deposition, gas-aerosol exchange + components=[current_var+'_BURDEN',current_var+'_CHML',current_var+'_CHMP', + current_var+'_DDF',current_var+'_WDF', current_var+'_SF', current_var+'_CLXF', + current_var+'_DDFC',current_var+'_WDFC'] + + if current_var=='SO4': + # For SULF we also have AQS, nucleation, and strat-trop gas exchange + components.append(current_var+'_AQS') + components.append(current_var+'_NUCL') + components.append(current_var+'_GAEX') + components.remove(current_var+'_CHMP') + + #components.append(current_var+'_CLXF') # BRT - CLXF is added above. + if current_var == "SOA": + components.append(current_var+'_GAEX') + #End if - AEROSOLS + + else: # CHEMS + # Components are: burden, chemical loss, chemical prod, dry/wet deposition, + # surface emissions, elevated emissions, chemical tendency + # I always add Lightning NOx production when calculating O3 budget. + + components=[current_var+'_BURDEN',current_var+'_CHML',current_var+'_CHMP', + current_var+'_DDF',current_var+'_WDF', current_var+'_SF', current_var+'_CLXF', + current_var+'_TEND'] + + if current_var =="O3": + components.append(current_var+'_LNO') + # End if + # End if + msg = f"chem/aerosol tables: 'make_Dic_scn_var_comp'" + msg += f"\n\t Current CAM variable: {current_var}" + msg += f"\n\t Derived components for CAM variable {current_var}: {components}" + #adfobj.debug_log(msg) + Dic_comp={} + Dic_comp,missing_vars,needed_vars=SEbudget(adfobj,dic_SE,current_dir,current_files,components,ext1_SE) + + for comp in components: + # Write details to log file + msg += f"\n\t\t calculate derived component: {comp} for main variable, {current_var}" + adfobj.debug_log(msg) + + # Gather info for debugging + for var_m in missing_vars: + if var_m not in missing_vars_tot: + missing_vars_tot.append(var_m) + for var_n in needed_vars: + if var_n not in needed_vars_tot: + needed_vars_tot.append(var_n) + # End for + # End for + # Set dictionary for key of current variable with dictionary values of all + # necessary constituents for calculating the current variable + Dic_var_comp[current_var] = Dic_comp + Dic_scn_var_comp = Dic_var_comp + + # Critical threshholds, just run this once + # this is for finding tropospheric values + # Critical threshholds?\n", + # Just run this once\n", + tropospheric_method='NA' + try: + current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['O3'],ext1_SE) + Dic_crit=current_crit['O3'] + tropospheric_method='ozone' + msg += f"\n\t WARNING: Troposphere is defined as O3<150 ppb" + + except: + try: + current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['TROP_P','Pressure'],ext1_SE) + Dic_crit=current_crit #[['TROP_P','Pressure']] + tropospheric_method='tropopause' + msg += f"\n\t WARNING: Troposphere is defined as pressure>trop_p" + except: + current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['U'],ext1_SE) + Dic_crit=current_crit['U'] + Tropospheric=False + msg += f"\n\t WARNING: No way of defining troposphere was found in the model, budgets are total column" + # Log info to logging file + msg = f"chem/aerosol tables:" + msg += f"\n\t - potential missing variables from budget? {missing_vars_tot}" + adfobj.debug_log(msg) + + msg = f"chem/aerosol tables:" + msg += f"\n\t - needed variables for budget {needed_vars_tot}" + adfobj.debug_log(msg) + + return Dic_crit,Dic_scn_var_comp,Tropospheric,tropospheric_method +##### + + +def SEbudget(adfobj,dic_SE,data_dir,files,vars,ext1_SE,**kwargs): + """ + Function used for getting the data for the budget calculation. This is the + chunk of code that takes the longest by far. + + Example: + ** This is for both chemistry and aeorosl calculations + + dic_SE: dictionary specyfing what variables to get. For example, + for precipitation you can define SE as: + dic_SE['PRECT']={'PRECC'+ext1_SE:8.64e7,'PRECL'+ext1_SE:8.64e7} + - It means to sum the file variables "PRECC" and "PRECL" + for my arbitrary desired variable named "PRECT" + + - It also has the option to apply conversion factors. + For instance, PRECL and PRECC are in m/s. 8.64e7 is used to convernt m/s to mm/day + + + data_dir: string of the directory that contains the files. always end with '/' + + files: list of the files to be read + + var: string showing the variable to be extracted. + -> this will be the individual componnent, ie O3_CHMP, SOA_WDF, etc. + """ + + # gas constanct + Rgas=287.04 #[J/K/Kg]=8.314/0.028965 + + # Set lists to gather necessary variables for logging + missing_vars = [] + needed_vars = [] + Dic_all_data={} + +# all_data=[] + for file in range(len(files)): + + ds=xr.open_dataset(Path(data_dir) / files[file]) + + # Calculate these just once + if file==0: + mock_2d=np.zeros_like(np.array(ds['PS'+ext1_SE].isel(time=0))) + mock_3d=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) + + try: + delP=np.array(ds['PDELDRY'+ext1_SE].isel(time=0)) + except: + + hyai=np.array(ds['hyai']) + hybi=np.array(ds['hybi']) + + try: + PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) + except: + PS=np.array(ds['PS'+ext1_SE].isel(time=0)) + # End try/except + + P0=1e5 + Plevel=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) + + for i in range(len(Plevel)): + Plevel[i]=hyai[i]*P0+hybi[i]*PS + + delP=Plevel[1:]-Plevel[:-1] + + for var in vars: + if file == 0: + Dic_all_data[var]=[] + + + # Star gathering of variable data + + if var=='TROP_P': + data=np.array(ds['TROP_P'+ext1_SE].isel(time=0))/100 + elif var== 'Pressure': + try: + data=np.array(ds['PMID'+ext1_SE].isel(time=0))/100 + except: + hyam=np.array(ds['hyam']) + hybm=np.array(ds['hybm']) + try: + PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) + except: + PS=np.array(ds['PS'+ext1_SE].isel(time=0)) + P0=1e5 + data=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) + for i in range(len(data)): + data[i]=hyam[i]*P0+hybm[i]*PS + data=data/100 + else: + + + data=[] + for i in dic_SE[var].keys(): + + if file == 0: + msg = f"chem/aerosol tables: 'SEbudget'" + msg += f"\n\t\t ** variable(s) needed for derived var {var}: {dic_SE[var].keys()}" + msg += f"\n\t\t - constituent for derived var {var}: {i}" + adfobj.debug_log(msg) + if i not in needed_vars: + needed_vars.append(i) + if ((i!='PS'+ext1_SE) and (i!='U'+ext1_SE) ) : + data.append(np.array(ds[i].isel(time=0))*dic_SE[var][i]) + else: + if i=='PS'+ext1_SE: + data.append(mock_2d) + else: + data.append(mock_3d) + # End if + if file == 0: + + if var not in missing_vars: + if var!='U': # This is to avoid confusion between U variable or U mock! + missing_vars.append(var) + msg += f"\n\t\t - no variable was found for var {var}: {i}" + + # End if + + # Get total summed data for this history file data + data=np.sum(data,axis=0) + # End try/except + + if ('CHML' in var) or ('CHMP' in var) : + Temp=np.array(ds['T'+ext1_SE].isel(time=0)) + try: + Pres=np.array(ds['PMID'+ext1_SE].isel(time=0)) + except: + hyam=np.array(ds['hyam']) + hybm=np.array(ds['hybm']) + try: + PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) + except: + PS=np.array(ds['PS'+ext1_SE].isel(time=0)) + P0=1e5 + Pres=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) + for i in range(len(Pres)): + Pres[i]=hyam[i]*P0+hybm[i]*PS + rho= Pres/(Rgas*Temp) + data=data*delP/rho + elif ('BURDEN' in var): + data=data*delP + else: + data=data + # End if + # Add data to list + Dic_all_data[var].append(data) + ds.close() + for var in vars: # Take mean + Dic_all_data[var]=np.nanmean(Dic_all_data[var],axis=0) + + + return Dic_all_data,missing_vars,needed_vars +##### + + +def calc_budget_data(current_var, Dic_scn_var_comp, area, trop, inside, num_yrs, duration, AEROSOLS): + """ + Function to run through desired table values for calculations for the table entries + """ + + # Initialize full data dictionary for current table type + chem_dict = {} + + # Update variable marker if neccessary + if current_var == 'SO4': + specifier = ' S' + else: + specifier = '' + + # Calculate values for given variable + if 'AOD' in current_var: + # Burden + spc_burd = Dic_scn_var_comp[current_var][current_var+'_AOD'] + burden = np.ma.masked_where(inside==False,spc_burd) #convert Kg/m2 to Tg + BURDEN = np.ma.sum(burden*area)/np.ma.sum(area) + chem_dict[f"{current_var}_mean"] = np.round(BURDEN,5) + else: + # Surface Emissions + spc_sf = Dic_scn_var_comp[current_var][current_var+'_SF'] + tmp_sf = spc_sf + sf = np.ma.masked_where(inside==False,tmp_sf*area) #convert Kg/m2/s to Tg/yr + SF = np.ma.sum(sf*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_EMIS (Tg{specifier}/yr)"] = np.round(SF,5) + + # Elevated Emissions + spc_clxf = Dic_scn_var_comp[current_var][current_var+'_CLXF'] + tmp_clxf = spc_clxf + clxf = np.ma.masked_where(inside==False,tmp_clxf*area) #convert Kg/m2/s to Tg/yr + CLXF = np.ma.sum(clxf*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_EMIS_elevated (Tg{specifier}/yr)"] = np.round(CLXF,5) + + # Burden + spc_burd = Dic_scn_var_comp[current_var][current_var+'_BURDEN'] + spc_burd = np.where(np.isnan(trop),np.nan,spc_burd) + tmp_burden = np.nansum(spc_burd*area,axis=0) + burden = np.ma.masked_where(inside==False,tmp_burden) #convert Kg/m2 to Tg + BURDEN = np.ma.sum(burden*1e-9) + chem_dict[f"{current_var}_BURDEN (Tg{specifier})"] = np.round(BURDEN,5) + + # Chemical Loss + spc_chml = Dic_scn_var_comp[current_var][current_var+'_CHML'] + spc_chml = np.where(np.isnan(trop),np.nan,spc_chml) + tmp_chml = np.nansum(spc_chml*area,axis=0) + chml = np.ma.masked_where(inside==False,tmp_chml) #convert Kg/m2/s to Tg/yr + CHML = np.ma.sum(chml*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_CHEM_LOSS (Tg{specifier}/yr)"] = np.round(CHML,5) + + # Chemical Production + if current_var == 'SO4': # chemical production is basically the elevated emissions. + # We have removed it for SO4 budget. and put 0 here, so, we don't report it + chem_dict[f"{current_var}_CHEM_PROD (Tg{specifier}/yr)"] = 0 + else: + spc_chmp = Dic_scn_var_comp[current_var][current_var+'_CHMP'] + spc_chmp = np.where(np.isnan(trop),np.nan,spc_chmp) + tmp_chmp = np.nansum(spc_chmp*area,axis=0) + chmp = np.ma.masked_where(inside==False,tmp_chmp) #convert Kg/m2/s to Tg/yr + CHMP = np.ma.sum(chmp*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_CHEM_PROD (Tg{specifier}/yr)"] = np.round(CHMP,5) + # End if + + # Aerosol calculations + #--------------------- + if current_var in AEROSOLS: + + # Dry Deposition Flux + spc_ddfa = Dic_scn_var_comp[current_var][current_var+'_DDF'] + spc_ddfc = Dic_scn_var_comp[current_var][current_var+'_DDFC'] + spc_ddf = spc_ddfa +spc_ddfc + tmp_ddf = spc_ddf + ddf = np.ma.masked_where(inside==False,tmp_ddf*area) #convert Kg/m2/s to Tg/yr + DDF = np.ma.sum(ddf*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_DRYDEP (Tg{specifier}/yr)"] = np.round(DDF,5) + + # Wet deposition + spc_wdfa = Dic_scn_var_comp[current_var][current_var+'_WDF'] + spc_wdfc = Dic_scn_var_comp[current_var][current_var+'_WDFC'] + spc_wdf = spc_wdfa +spc_wdfc + tmp_wdf = spc_wdf + wdf = np.ma.masked_where(inside==False,tmp_wdf*area) #convert Kg/m2/s to Tg/yr + WDF = -1*np.ma.sum(wdf*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_WETDEP (Tg{specifier}/yr)"] = np.round(WDF,5) + + if current_var in ["SOA",'SO4']: + # gas-aerosol Exchange + spc_gaex = Dic_scn_var_comp[current_var][current_var+'_GAEX'] + tmp_gaex = spc_gaex + gaex = np.ma.masked_where(inside==False,tmp_gaex*area) #convert Kg/m2/s to Tg/yr + GAEX = np.ma.sum(gaex*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_GAEX (Tg{specifier}/yr)"] = np.round(GAEX,5) + + # LifeTime = Burden/(loss+deposition) no chemical loss for aerosols + LT = BURDEN/(DDF+WDF)* duration/86400/num_yrs # days + chem_dict[f"{current_var}_LIFETIME (days)"] = np.round(LT,5) + + if current_var == 'SO4': + # Aqueous Chemistry + spc_aqs = Dic_scn_var_comp[current_var][current_var+'_AQS'] + tmp_aqs = spc_aqs + aqs = np.ma.masked_where(inside==False,tmp_aqs*area) #convert Kg/m2/s to Tg/yr + AQS = np.ma.sum(aqs*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_AQUEOUS (Tg{specifier}/yr)"] = np.round(AQS,5) + + # Nucleation + spc_nucl = Dic_scn_var_comp[current_var][current_var+'_NUCL'] + tmp_nucl = spc_nucl + nucl = np.ma.masked_where(inside==False,tmp_nucl*area) #convert Kg/m2/s to Tg/yr + NUCL = np.ma.sum(nucl*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_NUCLEATION (Tg{specifier}/yr)"] = np.round(NUCL,5) + + # Gaseous calculations + #--------------------- + else: + # Dry Deposition Flux + spc_ddf = Dic_scn_var_comp[current_var][current_var+'_DDF'] + tmp_ddf = spc_ddf + ddf = np.ma.masked_where(inside==False,tmp_ddf*area) #convert Kg/m2/s to Tg/yr + DDF = np.ma.sum(ddf*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_DRYDEP (Tg/yr)"] = np.round(DDF,5) + + # Wet Deposition Flux + spc_wdf = Dic_scn_var_comp[current_var][current_var+'_WDF'] + tmp_wdf = spc_wdf + wdf = np.ma.masked_where(inside==False,tmp_wdf*area) #convert Kg/m2/s to Tg/yr + WDF = -1*np.ma.sum(wdf*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_WETDEP (Tg/yr)"] = np.round(WDF,5) + + # Total Deposition + TDEP = DDF + WDF + chem_dict[f"{current_var}_TDEP (Tg/yr)"] = np.round(TDEP,5) + + # LifeTime = Burden/(loss+deposition) + if current_var == "CH4": + LT = BURDEN/(CHML+DDF+WDF) # years + chem_dict[f"{current_var}_LIFETIME (years)"] = np.round(LT,5) + else: + if (CHML+DDF+WDF) > 0: + if CHML != 0: + LT = BURDEN/(CHML+DDF+WDF)*duration/86400/num_yrs # days + chem_dict[f"{current_var}_LIFETIME (days)"] = np.round(LT,5) + else: + # do not report lifetime if chemical loss (for gases) is not included in the model outputs + # and put 0 here, so, we don't report it + chem_dict[f"{current_var}_LIFETIME (days)"] = 0 + # End if + # End if + # End if + + #NET = CHMP-CHML + # Chemical Tendency + spc_tnd = Dic_scn_var_comp[current_var][current_var+'_TEND'] + spc_tnd = np.where(np.isnan(trop),np.nan,spc_tnd) + tmp_tnd = np.nansum(spc_tnd,axis=0) + tnd = np.ma.masked_where(inside==False,tmp_tnd) #convert Kg/s to Tg/yr + TND = np.ma.sum(tnd*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_TEND (Tg/yr)"] = np.round(TND,5) + + # O3 dependent calculations + if current_var == "O3": + # Stratospheric-Tropospheric Exchange + STE = DDF-TND + chem_dict[f"{current_var}_STE (Tg/yr)"] = np.round(STE,5) + + # Lightning NOX production + spc_lno = Dic_scn_var_comp[current_var][current_var+'_LNO'] + tmp_lno = np.ma.masked_where(inside==False,spc_lno) + LNO = np.ma.sum(tmp_lno) + chem_dict[f"{current_var}_LNO (Tg N/yr)"] = np.round(LNO,5) + # End if (aerosol or gas) + return chem_dict +##### + + +def make_table(adfobj, vars, chem_type, Dic_scn_var_comp, areas, trops, case_names, nicknames, durations, insides, num_yrs, AEROSOLS): + """ + Create CSV table for aeorosols and gases, if applicable + + Table includes column values of variable, case(s), difference (if applicable) + + If this is a single model vs model run: 4 columns + first column: variables names, + second column: test case variable values + third column: baseline case variable values + final column: difference of test and baseline. + If this is a model vs obs run: 2 columns + first column: variables names, + second column: test case variable values + """ + # Initialize an empty dictionary to store DataFrames + dfs = {} + + #Special ADF variable which contains the output paths for + #all generated plots and tables for each case: + output_locs = adfobj.plot_location + + #Convert output location string to a Path object: + output_location = Path(output_locs[0]) + + # Loop over model cases + + for i,case in enumerate(nicknames): + + nickname = case + + # Collect row data in a list of dictionaries + #durations[case] + rows = [] + for current_var in vars: + chem_dict = calc_budget_data(current_var, Dic_scn_var_comp[case], areas[case], trops[case], insides[case], + num_yrs[case], durations[case], AEROSOLS) + # Loop through table variables + for key, val in chem_dict.items(): + if val != 0: # Skip variables with a value of 0 + print(f"\t - Variable '{key}' being added to table") + rows.append({'variable': key, nickname: np.round(val, 3)}) + elif 'OASISS_EMIS (' in key: # the paranthesis is to ignore EMIS_Elevated variables! + print(f"\t - Variable '{key}' being added to table") + rows.append({'variable': key, nickname: np.round(val, 3)}) + else: + msg = f"chem/aerosol tables:" + msg += f"\n\t - Variable '{key}' has value of 0, will not add to table" + adfobj.debug_log(msg) + # End if + # End for + # End for + + # Create the DataFrame for the current case + table_df = pd.DataFrame(rows) + + if chem_type == 'gases': + # Replace compound names directly in the DataFrame + replacements = { + 'MTERP': 'Monoterpene', + 'CH3OH': 'Methanol', + 'CH3COCH3': 'Acetone', + 'O3_LNO': 'LNOx_PROD' + } + table_df['variable'] = table_df['variable'].replace(replacements, regex=True) + # End if + + # Store the DataFrame in the dictionary + dfs[nickname] = table_df + + # End for + + # Merge the DataFrames on the 'variable' column + if len(case_names) == 2: + + table_df = pd.merge(dfs[nicknames[0]], dfs[nicknames[1]], on='variable') + + # Calculate the differences between case columns + table_df['difference'] = table_df[nicknames[0]] - table_df[nicknames[1]] + + #Create output file name: + output_csv_file = output_location / f'ADF_amwg_{chem_type}_table.csv' + # Save table to CSV and add table dataframe to website (if enabled) + table_df.to_csv(output_csv_file, index=False) + #adfobj.add_website_data(table_df, chem_type, case, plot_type="Tables") + adfobj.add_website_data(table_df, chem_type, case_names[0], plot_type="Tables") + +##### diff --git a/scripts/analysis/aerosol_gas_tables_Tropopause_version1.py b/scripts/analysis/aerosol_gas_tables_Tropopause_version1.py new file mode 100644 index 000000000..874f884b7 --- /dev/null +++ b/scripts/analysis/aerosol_gas_tables_Tropopause_version1.py @@ -0,0 +1,1379 @@ +import numpy as np +import xarray as xr +import sys +from pathlib import Path +import warnings # use to warn user about missing files. + +from datetime import datetime +import numpy as np +import itertools + +try: + import pandas as pd +except ImportError: + print("Pandas module does not exist in python path, but is needed for amwg_table.") + print("Please install module, e.g. 'pip install pandas'.") + sys.exit(1) +#End except + +# Import necessary ADF modules: +from adf_base import AdfError + +def aerosol_gas_tables(adfobj): + ''' + Calculate aerosol and gaseous budget tables + + Default set of variables: change in lib/adf_variable_defaults.yaml + ------------------------- + GAS_VARIABLES: ['CH4','CH3CCL3', 'CO', 'O3', 'ISOP', 'MTERP', 'CH3OH', 'CH3COCH3'] + AEROSOL_VARIABLES: ['AOD','SOA', 'SALT', 'DUST', 'POM', 'BC', 'SO4'] + + Default output for tables: + + Gases: + ------ + CH4_BURDEN (Tg), CH4_CHEM_LOSS (Tg/yr), CH4_LIFETIME (years) + + CH3CCL3_BURDEN (Tg), CH3CCL3_CHEM_LOSS (Tg/yr), CH3CCL3_LIFETIME (days) + + CO_EMIS (Tg/yr), CO_BURDEN (Tg), CO_CHEM_LOSS (Tg/yr), CO_CHEM_PROD (Tg/yr), CO_DRYDEP (Tg/yr) + CO_TDEP (Tg/yr), CO_LIFETIME (days), CO_TEND (Tg/yr) + + O3_BURDEN (Tg), O3_CHEM_LOSS (Tg/yr), O3_CHEM_PROD (Tg/yr), O3_DRYDEP (Tg/yr), O3_TDEP (Tg/yr) + O3_LIFETIME (days), O3_TEND (Tg/yr), O3_STE (Tg/yr) + + LNOx_PROD (Tg N/yr) + + ISOP_EMIS (Tg/yr), ISOP_BURDEN (Tg) + + Monoterpene_EMIS (Tg/yr), Monoterpene_BURDEN (Tg) + + Methanol_EMIS (Tg/yr), Methanol_BURDEN (Tg), Methanol_DRYDEP (Tg/yr), Methanol_WETDEP (Tg/yr), Methanol_TDEP (Tg/yr) + + Acetone_EMIS (Tg/yr), Acetone_BURDEN (Tg), Acetone_DRYDEP (Tg/yr), Acetone_WETDEP (Tg/yr), Acetone_TDEP (Tg/yr) + + + + Aerosols: + --------- + AOD_mean + + SOA_BURDEN (Tg), SOA_CHEM_LOSS (Tg/yr), SOA_DRYDEP (Tg/yr), SOA_WETDEP (Tg/yr), SOA_GAEX (Tg/yr), SOA_LIFETIME (days) + + SALT_EMIS (Tg/yr), SALT_BURDEN (Tg), SALT_DRYDEP (Tg/yr), SALT_WETDEP (Tg/yr), SALT_LIFETIME (days) + + DUST_EMIS (Tg/yr), DUST_BURDEN (Tg), DUST_DRYDEP (Tg/yr), DUST_WETDEP (Tg/yr), DUST_LIFETIME (days) + + POM_EMIS (Tg/yr), POM_BURDEN (Tg), POM_DRYDEP (Tg/yr), POM_WETDEP (Tg/yr), POM_LIFETIME (days) + + BC_EMIS (Tg/yr), BC_BURDEN (Tg), BC_DRYDEP (Tg/yr), BC_WETDEP (Tg/yr), BC_LIFETIME (days) + + SO4_EMIS_elevated (Tg S/yr), SO4_BURDEN (Tg S), SO4_DRYDEP (Tg S/yr), SO4_WETDEP (Tg S/yr), SO4_GAEX (Tg S/yr) + SO4_LIFETIME (days), SO4_AQUEOUS (Tg S/yr), SO4_NUCLEATION (Tg S/yr) + + + List of variable names and descriptions for clarity + --------------------------------------------------- + - ListVars: list of all available variables from given history file + - GAS_VARIABLES: list fo necessary CAM gaseous variables + - AEROSOL_VARIABLES: list fo necessary CAM aerosol variables + - AEROSOLS: list of necessary aerosols for computations + + + MODIFICATION HISTORY: + Behrooz Roozitalab, 02, NOV, 2022: VERSION 1.00 + - Initial version + + Justin Richling, 27 Nov, 2023 + - updated to fit to ADF and check with old AMWG chem/aerosol tables + - fixed: + * added difference bewtween cases column to tables + + Behrooz Roozitalab, 8 Aug, 2024 + - fixed: + * lifetime inconsitencies + * Removed redundant calculations to improve the speed + * Verified the results against the NCL script. + + Behrooz Roozitalab, 5 Jun, 2025 + - fixed: + * Fix the bugs in the calculation (when converting from Jupyterhub to ADF) + * add the 'U' variable in dic_SE + * make the code faster by modifying make_Dic_scn_var_comp + * Add a condition to calculate whole world budgets when O3 is not find. + * Update pressure calculation in a more general way. + + Behrooz Roozitalab, 20 Aug, 2025 _ Version 1 + - fixed: + * the html page was not created, it is fixed. + * added "hm" as a case to enable using annual averaged files in addition to monthly files. + * This version uses 500hPa as the tropopause threshold. Use Version 0 for a realistic case. + * Added DMS to gases list - reported as DMS not S + * Automatic addition of gaseous compounds even when not defined in the default list, + * based on Carbon MW (12). It still needs ADF modification to read a list from yaml file. + ''' + + + #Notify user that script has started: + msg = "\n Calculating chemistry/aerosol budget tables..." + print(f"{msg}\n {'-' * (len(msg)-3)}") + + # Inputs + #------- + # Variable defaults info + res = adfobj.variable_defaults # dict of variable-specific plot preferences + bres = res['budget_tables'] + # list of the gaseous variables to be caculated. + GAS_VARIABLES = bres['GAS_VARIABLES'] + + # list of the aerosol variables to be caculated. + AEROSOL_VARIABLES = bres['AEROSOL_VARIABLES'] + + #list of all the variables to be caculated. + VARIABLES = GAS_VARIABLES + AEROSOL_VARIABLES + + # For the case that outputs are saved for a specific region. + # i.e., when using fincllonlat in user_nl_cam + ext1_SE = bres['ext1_SE'] + + # Tropospheric Values + # ------------------- + # if True, calculate only Tropospheric values + # if False, all layers + # tropopause is defiend as either directly or indirectly. Look for tropopause to see the definition + Tropospheric = bres['Tropospheric'] + + ### NOT WORKING FOR NOW + # To calculate the budgets only for a region + # Lat/Lon extent + limit = bres['limit'] + regional = bres['regional'] + + # Dictionary for Molecular weights. Keys must be consistent with variable name + # For aerosols, the MW is used only for chemical loss, chemical production, and elevated emission calculations + # For SO4, we report everything in terms of Sulfur, so we use Sulfur MW here + MW = bres['MW'] + + # automatic generation of MW + for var in VARIABLES: + if var not in MW.keys(): + print(f"using Carbon molecular weight for {var}") + MW[var]=12 + + + # Avogadro's Number + AVO = float(bres['AVO']) + # gravity + gr = float(bres['gr']) + # Mw air + Mwair = float(bres['Mwair']) + + # The variables in the list below must be aerosols - do not add AOD and DAOD + # no need to change this list, unless for a specific need! + AEROSOLS = bres['AEROSOLS'] + + # Start gathering case, path, and data info + #----------------------------------------- + + # CAM simulation variables (these quantities are always lists): + case_names = adfobj.get_cam_info('cam_case_name', required=True) + + # Grab all case nickname(s) + test_nicknames_list = adfobj.case_nicknames["test_nicknames"] + nicknames_list = test_nicknames_list + # Grab climo years + start_years = adfobj.climo_yrs["syears"] + end_years = adfobj.climo_yrs["eyears"] + + #Grab history strings: + hist_strs = adfobj.hist_string["test_hist_str"] + + # Grab history file locations from config yaml file + hist_locs = adfobj.get_cam_info("cam_hist_loc", required=True) + + # Check if this is test model vs baseline model + # If so, update test case(s) lists created above + if not adfobj.compare_obs: + # Get baseline case info + case_names += [adfobj.get_baseline_info("cam_case_name")] + nicknames_list += [adfobj.case_nicknames["base_nickname"]] + + # Grab climo years + start_years += [adfobj.climo_yrs["syear_baseline"]] + end_years += [adfobj.climo_yrs["eyear_baseline"]] + + # Get history file info + hist_strs += [adfobj.hist_string["base_hist_str"]] + hist_locs += [adfobj.get_baseline_info("cam_hist_loc")] + # End if + + # Check to ensure number of case names matches number history file locations. + # If not, exit script + if len(hist_locs) != len(case_names): + errmsg = "Error: number of cases does not match number of history file locations. Script is exiting." + raise AdfError(errmsg) + + # Initialize nicknames dictionary + #nicknames = {} + + # Filter the list to include only strings that are possible h0 strings + # - Search for either h0 or h0a + substrings = {"cam.h0","cam.h0a","cam.hm"} + case_hist_strs = [] + for cam_case_str in hist_strs: + # Check each possible h0 string + for string in cam_case_str: + if string in substrings: + case_hist_strs.append(string) + break + + # Create path object for the CAM history file(s) location: + data_dirs = [] + for case_idx,case in enumerate(nicknames_list): + + print(f"\t Looking for history location: {hist_locs[case_idx]}") + + + #Check that history file input directory actually exists: + if (hist_locs[case_idx] is None) or (not Path(hist_locs[case_idx]).is_dir()): + errmsg = f"History files directory '{hist_locs[case_idx]}' not found. Script is exiting." + raise AdfError(errmsg) + + #Write to debug log if enabled: + adfobj.debug_log(f"DEBUG: location of history files is {str(hist_locs[case_idx])}") + # Update list for found directories + data_dirs.append(hist_locs[case_idx]) + + # End gathering case, path, and data info + #----------------------------------------- + # Periods of Interest + # ------------------- + # choose the period of interest. Plots will be averaged within this period + durations = {} + num_yrs = {} + + # Main function + #-------------- + # Set dictionary of components for each case + Dic_scn_var_comp = {} + areas = {} + trops = {} + insides = {} + for i,case in enumerate(nicknames_list): + + start_year = start_years[i] + end_year = end_years[i] + 1 + start_date = f"{start_year}-1-1" + end_date = f"{end_year}-1-1" + + # Create time periods + start_period = datetime.strptime(start_date, "%Y-%m-%d") + end_period = datetime.strptime(end_date, "%Y-%m-%d") + + # Calculated duration of time period in seconds? + durations[case] = (end_period-start_period).days*86400 #+365*86400 + + + # Get number of years for calculations + num_yrs[case] = (int(end_year)-int(start_year)) #+1 + + # Get currenty history file directory + data_dir = data_dirs[i] + + # Get all files, lats, lons, and area weights for current case + Files,Lats,Lons,areas[case],ext1_SE = Get_files(adfobj,data_dir,start_year,end_year,case_hist_strs[i],area=True) + # find the name of all the variables in the file. + # this will help the code to work for the variables that are not in the files (assingn 0s) + tmp_file = xr.open_dataset(Path(data_dir) / Files[0]) + ListVars = list(tmp_file.variables) + + # Set up and fill dictionaries for components for current cases + dic_SE = set_dic_SE(ListVars,ext1_SE,VARIABLES) + dic_SE = fill_dic_SE(adfobj, dic_SE, VARIABLES, ListVars, ext1_SE, AEROSOLS, MW, AVO, gr, Mwair) + + text = f'\n\t Calculating values for {case}' + print(text) + print("\t " + "-" * (len(text) - 2)) + + # Gather dictionary data for current case + # NOTE: The calculations can take a long time... + Dic_crit, Dic_scn_var_comp[case],Tropospheric,tropospheric_method = make_Dic_scn_var_comp(adfobj, VARIABLES, data_dir, dic_SE, Files, ext1_SE, AEROSOLS,Tropospheric) + # Regional refinement + # NOTE: This function 'Inside_SE' is unavailable at the moment! - JR 10/2024 + if regional: + #inside = Inside_SE_region(current_lat,current_lon,dir_shapefile) + inside = Inside_SE(Lats,Lons,limit) + else: + if len(np.shape(areas[case])) == 1: + inside = np.full((len(Lons)),True) + else: + inside = np.full((len(Lats),len(Lons)),True) + + # Set critical threshold + current_crit = Dic_crit + if Tropospheric: + if tropospheric_method=='pressure': + # using pressure > 500hPa + trop = np.where(current_crit<500,np.nan,current_crit) + elif tropospheric_method=='NA': + print('ERROR: Tropopause is not defined correctly!') + else: + trop=current_crit + trops[case] = trop + insides[case] = inside + + # Make and save tables + table_kwargs = {"adfobj":adfobj, + "Dic_scn_var_comp":Dic_scn_var_comp, + "areas":areas, + "trops":trops, + "case_names":case_names, + "nicknames":nicknames_list, + "durations":durations, + "insides":insides, + "num_yrs":num_yrs, + "AEROSOLS":AEROSOLS} + + #print(table_kwargs) + + # Create the budget tables + #------------------------- + # Aerosols + if len(AEROSOL_VARIABLES) > 0: + print("\tMaking table for aerosols") + make_table(vars=AEROSOL_VARIABLES, chem_type='aerosols', **table_kwargs) + # Gases + if len(GAS_VARIABLES) > 0: + print("\tMaking table for gases") + make_table(vars=GAS_VARIABLES, chem_type='gases', **table_kwargs) +####### + +################## +# Helper functions +################## + +def list_files(adfobj, directory, start_year ,end_year, h_case): + + """ + This function extracts the files in the directory that are within the chosen dates + and history number. + """ + + # History file year range + yrs = np.arange(int(start_year), int(end_year)) + + all_filenames = [] + for i in yrs: +# all_filenames.append(sorted(Path(directory).glob(f'*.{h_case}.{i}-*'))) + all_filenames.append(sorted(Path(directory).glob(f'*.{h_case}.{i}*'))) + + #print(directory) + # Flattening the list of lists + filenames = list(itertools.chain.from_iterable(sorted(all_filenames))) + if len(filenames)==0: + #sys.exit(" Directory has no outputs ") + msg = f"chem/aerosol tables, 'list_files':" + msg += f"\n\t - Directory '{directory}' has no outputs." + adfobj.debug_log(msg) + + return filenames +##### + + +def Get_files(adfobj, data_dir, start_year, end_year, h_case, **kwargs): + + """ + This function retrieves the files, latitude, and longitude information + in all the directories within the chosen dates. + """ + ext1_SE = kwargs.pop('ext1_SE','') + area = kwargs.pop('area',False) + + Earth_rad=6.371e6 # Earth Radius in meters + + current_files = list_files(adfobj, data_dir, start_year, end_year,h_case) + # get the Lat and Lons for each case + tmp_file = xr.open_dataset(Path(data_dir) / current_files[0]) + lon = tmp_file['lon'+ext1_SE].data + lon[lon > 180.] -= 360 # shift longitude from 0-360˚ to -180-180˚ + lat = tmp_file['lat'+ext1_SE].data + + if area == True: + try: + tmp_area = tmp_file['area'+ext1_SE].data + Earth_area = 4 * np.pi * Earth_rad**(2) + + areas = tmp_area*Earth_area/np.nansum(tmp_area) + except KeyError: + try: + tmp_area = tmp_file['AREA'+ext1_SE].isel(time=0).data + areas=tmp_area + #Earth_area = 4 * np.pi * Earth_rad**(2) + #areas = tmp_area*Earth_area/np.nansum(tmp_area) + except: + dlon = np.abs(lon[1]-lon[0]) + dlat = np.abs(lat[1]-lat[0]) + + lon2d,lat2d = np.meshgrid(lon,lat) + #area=np.zeros_like(lat2d) + + dy = Earth_rad*dlat*np.pi/180 + dx = Earth_rad*np.cos(lat2d*np.pi/180)*dlon*np.pi/180 + + tmp_area = dx*dy + areas = tmp_area + # End if + + # Variables to return + return current_files,lat,lon,areas,ext1_SE +##### + +def set_dic_SE(ListVars, ext1_SE,variables): + """ + Initialize dictionary to house all the relevant tabel data + """ + + # Initialize dictionary + #---------------------- + dic_SE={} + + # Chemistry + #---------- + dic_SE['U']={'U'+ext1_SE:1} + dic_SE['O3']={'O3'+ext1_SE:1e9} # covert to ppb for Tropopause calculation + dic_SE['CH4']={'CH4'+ext1_SE:1} + dic_SE['CO']={'CO'+ext1_SE:1} + + dic_SE['ISOP']={'ISOP'+ext1_SE:1} + dic_SE['MTERP']={'MTERP'+ext1_SE:1} + dic_SE['CH3OH']={'CH3OH'+ext1_SE:1} + dic_SE['CH3COCH3']={'CH3COCH3'+ext1_SE:1} + dic_SE['CH3CCL3']={'CH3CCL3'+ext1_SE:1} + dic_SE['CHBR3']={'CHBR3'+ext1_SE:1} + dic_SE['CH2BR2']={'CH2BR2'+ext1_SE:1} + + # Aerosols + #--------- + + dic_SE['DAOD']={'AODDUSTdn'+ext1_SE:1} + dic_SE['AOD']={'AODVISdn'+ext1_SE:1} + + dic_SE['DUST']={'dst_a1'+ext1_SE:1, + 'dst_a2'+ext1_SE:1, + 'dst_a3'+ext1_SE:1} + + dic_SE['SALT']={'ncl_a1'+ext1_SE:1, + 'ncl_a2'+ext1_SE:1, + 'ncl_a3'+ext1_SE:1} + + dic_SE['POM']={'pom_a1'+ext1_SE:1, + 'pom_a4'+ext1_SE:1} + + dic_SE['BC']={'bc_a1'+ext1_SE:1, + 'bc_a4'+ext1_SE:1} + + + dic_SE['SO4']={'so4_a1'+ext1_SE:1, + 'so4_a2'+ext1_SE:1, + 'so4_a3'+ext1_SE:1, + 'so4_a5'+ext1_SE:1} + + # FOR SOA, first check if the integrated bins are included + if (('soa_a1'+ext1_SE in ListVars ) & ('soa_a1'+ext1_SE in ListVars )): + dic_SE['SOA'] = {'soa_a1'+ext1_SE:1, + 'soa_a2'+ext1_SE:1} + else: + dic_SE['SOA'] = {'soa1_a1'+ext1_SE:1, + 'soa2_a1'+ext1_SE:1, + 'soa3_a1'+ext1_SE:1, + 'soa4_a1'+ext1_SE:1, + 'soa5_a1'+ext1_SE:1, + 'soa1_a2'+ext1_SE:1, + 'soa2_a2'+ext1_SE:1, + 'soa3_a2'+ext1_SE:1, + 'soa4_a2'+ext1_SE:1, + 'soa5_a2'+ext1_SE:1} + + dic_SE['DMS']={'DMS'+ext1_SE:1} + #dic_SE['TROP_P']={'TROP_P'+ext1_SE:1} + + + # automatic generation of dic_SE + for var in variables: + if var not in dic_SE.keys(): + dic_SE[var]={var+ext1_SE:1} + + # consider for OASISS DMS separately + if var=='DMS': + dic_SE['DMS_OASISS']={'DMS_OASISS'+ext1_SE:1} + # End if + + return dic_SE +##### + +def fill_dic_SE(adfobj, dic_SE, variables, ListVars, ext1_SE, AEROSOLS, MW, AVO, gr, Mwair): + """ + Function for dealing with conversion factors for different components and filling the main data + dictionary 'dic_SE' + + Input dictionary and return updated dictionary 'dic_SE' + + Arguments + --------- + variables : list + - list of main variables? + ListVars : list + - list of ??????? + + Returns + ------- + dic_SE : dict + - full dictionary of derived variables + + Some conversion factors need density or Layer's pressure, that will be accounted for when reading the files. + We convert everying to kg/m2/s or kg/m2 or kg/s, so that final Tg/yr or Tg results are consistent + """ + + # Logging info message + msg = f"chem/aerosol tables: 'fill_dic_SE'" + + for var in variables: + + if 'AOD' in var: + dic_SE[var+'_AOD']={} + else: + dic_SE[var+'_BURDEN']={} + dic_SE[var+'_CHML']={} + dic_SE[var+'_CHMP']={} + + dic_SE[var+'_SF']={} + dic_SE[var+'_CLXF']={} + + dic_SE[var+'_DDF']={} + dic_SE[var+'_WDF']={} + + if var in AEROSOLS: + dic_SE[var+'_GAEX']={} + dic_SE[var+'_DDFC']={} + dic_SE[var+'_WDFC']={} + else: + dic_SE[var+'_TEND']={} + dic_SE[var+'_LNO']={} + # End if + + # We have nucleation and aqueous chemistry for sulfate. + if var=='SO4': + dic_SE[var+'_NUCL']={} + dic_SE[var+'_AQS']={} + # End if + + # Grab the variable keys + var_keys = dic_SE[var].keys() + + for key in var_keys: + msg += f"\n\t Creating component of {var}: {key}" + + # for CHML and CHMP: + # original unit : [molec/cm3/s] + # following Tilmes code to convert to [kg/m2/s] + # conversion: Mw*rho*delP*1e3/Avo/gr + # rho and delP will be applied when reading the files in SEbudget function. + + # for AOD and DAOD: + if 'AOD' in var: + if key in ListVars: + dic_SE[var+'_AOD'][key+ext1_SE]=1 + else: + dic_SE[var+'_AOD']['PS'+ext1_SE]=0. + # End if + continue # AOD doesn't need any other budget calculations + # End if + + # for CHML and CHMP: + # original unit : [molec/cm3/s] + # following Tilmes code to convert to [kg/m2/s] + # conversion: Mw*rho*delP*1e3/Avo/gr + # rho and delP will be applied when reading the files in SEbudget function. + if key=='O3'+ext1_SE: + # for O3, we should not include fast cycling reactions + # As a result, we use below diagnostics in the model if available. If not, we use CHML and CHMP + if ((key+'_Loss' in ListVars) & (key+'_Prod' in ListVars)) : + dic_SE[var+'_CHML'][key+'_Loss'+ext1_SE]=MW[var]*1e3/AVO/gr + dic_SE[var+'_CHMP'][key+'_Prod'+ext1_SE]=MW[var]*1e3/AVO/gr + else: + if key+'_CHML' in ListVars: + dic_SE[var+'_CHML'][key+'_CHML'+ext1_SE]=MW[var]*1e3/AVO/gr + else: + dic_SE[var+'_CHML']['U'+ext1_SE]=0 + # End if + + if key+'_CHMP' in ListVars: + dic_SE[var+'_CHMP'][key+'_CHMP'+ext1_SE]=MW[var]*1e3/AVO/gr + else: + dic_SE[var+'_CHMP']['U'+ext1_SE]=0 + # End if + # End if + else: + if key+'_CHML' in ListVars: + dic_SE[var+'_CHML'][key+'_CHML'+ext1_SE]=MW[var]*1e3/AVO/gr + else: + dic_SE[var+'_CHML']['U'+ext1_SE]=0 + # End if + + if key+'_CHMP' in ListVars: + dic_SE[var+'_CHMP'][key+'_CHMP'+ext1_SE]=MW[var]*1e3/AVO/gr + else: + dic_SE[var+'_CHMP']['U'+ext1_SE]=0 + # End if + # End if + + + # for SF: + # original unit: [kg/m2/s] + if 'SF'+key in ListVars: + if var=='SO4': + dic_SE[var+'_SF']['SF'+key+ext1_SE]=32.066/115.11 + else: + dic_SE[var+'_SF']['SF'+key+ext1_SE]=1 + elif ((var=='DMS_OASISS') & ('OCN_FLUX_DMS' in ListVars)): + dic_SE[var+'_SF']['OCN_FLUX_DMS'+ext1_SE]=1 + + # End if + elif key+'SF' in ListVars: + dic_SE[var+'_SF'][key+ext1_SE+'SF']=1 + else: + dic_SE[var+'_SF']['PS'+ext1_SE]=0. + # End if + + + # for CLXF: + # original unit: [molec/cm2/s] + # conversion: Mw*10/Avo + if key+'_CLXF' in ListVars: + dic_SE[var+'_CLXF'][key+'_CLXF'+ext1_SE]=MW[var]*10/AVO # convert [molec/cm2/s] to [kg/m2/s] + else: + dic_SE[var+'_CLXF']['PS'+ext1_SE]=0. + # End if + + # Aerosols + if var in AEROSOLS: + # for each species: + # original unit : [kg/kg] in dry air + # convert to [kg/m2] + # conversion: delP/gr + # delP will be applied when reading the files in SEbudget function. + if key in ListVars: + if var=='SO4': # For SO4, we report all the budget calculation for Sulfur. + dic_SE[var+'_BURDEN'][key+ext1_SE]=(32.066/115.11)/gr + else: + dic_SE[var+'_BURDEN'][key+ext1_SE]=1/gr + # End if + else: + dic_SE[var+'_BURDEN']['U'+ext1_SE]=0 + # End if + + + # for DDF: + # original unit: [kg/m2/s] + if key+'DDF' in ListVars: + if var=='SO4': + dic_SE[var+'_DDF'][key+ext1_SE+'DDF']=32.066/115.11 + else: + dic_SE[var+'_DDF'][key+ext1_SE+'DDF']=1 + # End if + else: + dic_SE[var+'_DDF']['PS'+ext1_SE]=0. + # End if + + + # for SFWET: + # original unit: [kg/m2/s] + if key+'SFWET' in ListVars: + if var=='SO4': + dic_SE[var+'_WDF'][key+ext1_SE+'SFWET']=32.066/115.11 + else: + dic_SE[var+'_WDF'][key+ext1_SE+'SFWET']=1 + # End if + else: + dic_SE[var+'_WDF']['PS'+ext1_SE]=0. + # End if + + + # for sfgaex1: + # original unit: [kg/m2/s] + if key+'_sfgaex1' in ListVars: + if var=='SO4': + dic_SE[var+'_GAEX'][key+ext1_SE+'_sfgaex1']=32.066/115.11 + else: + dic_SE[var+'_GAEX'][key+ext1_SE+'_sfgaex1']=1 + # End if + else: + dic_SE[var+'_GAEX']['PS'+ext1_SE]=0. + # End if + + + # for DDF in cloud water: + # original unit: [kg/m2/s] + cloud_key=key[:-2]+'c'+key[-1] + if cloud_key+ext1_SE+'DDF' in ListVars: + if var=='SO4': + dic_SE[var+'_DDFC'][cloud_key+ext1_SE+'DDF']=32.066/115.11 + else: + dic_SE[var+'_DDFC'][cloud_key+ext1_SE+'DDF']=1 + # End if + else: + dic_SE[var+'_DDFC']['PS'+ext1_SE]=0. + # End if + + # for SFWET in cloud water: + # original unit: [kg/m2/s] + if cloud_key+ext1_SE+'SFWET' in ListVars: + if var=='SO4': + dic_SE[var+'_WDFC'][cloud_key+ext1_SE+'SFWET']=32.066/115.11 + else: + dic_SE[var+'_WDFC'][cloud_key+ext1_SE+'SFWET']=1 + # End if + else: + dic_SE[var+'_WDFC']['PS'+ext1_SE]=0. + # End if + + if var=='SO4': + # for Nucleation : + # original unit: [kg/m2/s] + if key+ext1_SE+'_sfnnuc1' in ListVars: + dic_SE[var+'_NUCL'][key+ext1_SE+'_sfnnuc1']=32.066/115.11 + else: + dic_SE[var+'_NUCL']['PS'+ext1_SE]=0. + # End if + + # for Aqueous phase : + # original unit: [kg/m2/s] + if (('AQSO4_H2O2'+ext1_SE in ListVars) & ('AQSO4_O3'+ext1_SE in ListVars)) : + dic_SE[var+'_AQS']['AQSO4_H2O2'+ext1_SE]=32.066/115.11 + dic_SE[var+'_AQS']['AQSO4_O3'+ext1_SE]=32.066/115.11 + else: + # original unit: [kg/m2/s] + if cloud_key+'AQSO4'+ext1_SE in ListVars: + dic_SE[var+'_AQS'][cloud_key+'AQSO4'+ext1_SE]=32.066/115.11 + else: + dic_SE[var+'_AQS']['PS'+ext1_SE]=0. + # End if + + if cloud_key+'AQH2SO4'+ext1_SE in ListVars: + dic_SE[var+'_AQS'][cloud_key+'AQH2SO4'+ext1_SE]=32.066/115.11 + else: + dic_SE[var+'_AQS']['PS'+ext1_SE]=0. + # End if + # End if + # End if + + else: # Gases + # for each species: + # original unit : [mole/mole] in dry air + # convert to [kg/m2] + # conversion: Mw*delP/Mwair/gr Mwair=28.97 gr/mole + # delP will be applied when reading the files in SEbudget function. + if key in ListVars: + dic_SE[var+'_BURDEN'][key+ext1_SE]=MW[var]/Mwair/gr + else: + dic_SE[var+'_BURDEN']['U'+ext1_SE]=0 + # End if + + # for DF: + # original unit: [kg/m2/s] + if 'DF_'+key in ListVars: + dic_SE[var+'_DDF']['DF_'+key+ext1_SE]=1 + else: + dic_SE[var+'_DDF']['PS'+ext1_SE]=0. + # End if + + # for WD: + # original unit: [kg/m2/s] + if 'WD_'+key in ListVars: + dic_SE[var+'_WDF']['WD_'+key+ext1_SE]=1 + else: + dic_SE[var+'_WDF']['PS'+ext1_SE]=0. + # End if + + # for Chem tendency: + # original unit: [kg/s] + # conversion: not needed + if 'D'+key+'CHM' in ListVars: + dic_SE[var+'_TEND']['D'+key+'CHM'+ext1_SE]=1 # convert [kg/s] to [kg/s] + else: + dic_SE[var+'_TEND']['U'+ext1_SE]=0 + # End if + + # for Lightning NO production: (always in gas) + # original unit: [Tg N/Yr] + # conversion: not needed + if 'LNO_COL_PROD' in ListVars: + dic_SE[var+'_LNO']['LNO_COL_PROD'+ext1_SE]=1 # convert [Tg N/yr] to [Tg N /yr] + else: + dic_SE[var+'_LNO']['PS'+ext1_SE]=0 + # End if + # End if (aerosols or gases) + # End for + # End for + + # Write to log + adfobj.debug_log(msg) + + return dic_SE +##### + + +def make_Dic_scn_var_comp(adfobj, variables, current_dir, dic_SE, current_files, ext1_SE, AEROSOLS,Tropospheric): + """ + This function retrieves the files, latitude, and longitude information + in all the directories within the chosen dates. + + current_dir: list + - showing the directories to look for files. always end with '/' + + current_files: list + - List of CAM history files + + start_year: string + - Starting year + + end_year: string + - Ending year + + kwargs + ------ + ext1_SE: string + - specify if the files are for only a region, which changes to variable names. + ex: if you saved files for a only a box region ($LL_lat$,$LL_lon$,$UR_lat$,$UR_lon$), + the 'lat' variable will be saved as: 'lat_$LL_lon$e_to_$UR_lon$e_$LL_lat$n_to_$UR_lat$n' + for instance: 'lat_65e_to_91e_20n_to_32n' + + Returns + ------- + Dic_crit: + - dictionary for critical values for current case + Dic_scn_var_comp: + - full dictionary of all variables and components for current case + + NOTE: The LNO is lightning NOx, which should be reported explicitly rather as CO_LNO, O3_LNO, ... + """ + + # Set lists to gather necessary variables for logging + missing_vars_tot = [] + needed_vars_tot = [] + + # Initialize final component dictionary + Dic_var_comp={} + + for current_var in variables: + if 'AOD' in current_var: + components=[current_var+'_AOD'] + else: + if current_var in AEROSOLS: # AEROSOLS + + # Components are: burden, chemical loss, chemical prod, dry deposition, + # surface emissions, elevated emissions, wet deposition, gas-aerosol exchange + components=[current_var+'_BURDEN',current_var+'_CHML',current_var+'_CHMP', + current_var+'_DDF',current_var+'_WDF', current_var+'_SF', current_var+'_CLXF', + current_var+'_DDFC',current_var+'_WDFC'] + + if current_var=='SO4': + # For SULF we also have AQS, nucleation, and strat-trop gas exchange + components.append(current_var+'_AQS') + components.append(current_var+'_NUCL') + components.append(current_var+'_GAEX') + components.remove(current_var+'_CHMP') + + #components.append(current_var+'_CLXF') # BRT - CLXF is added above. + if current_var == "SOA": + components.append(current_var+'_GAEX') + #End if - AEROSOLS + + else: # CHEMS + # Components are: burden, chemical loss, chemical prod, dry/wet deposition, + # surface emissions, elevated emissions, chemical tendency + # I always add Lightning NOx production when calculating O3 budget. + + components=[current_var+'_BURDEN',current_var+'_CHML',current_var+'_CHMP', + current_var+'_DDF',current_var+'_WDF', current_var+'_SF', current_var+'_CLXF', + current_var+'_TEND'] + + if current_var =="O3": + components.append(current_var+'_LNO') + # End if + # End if + msg = f"chem/aerosol tables: 'make_Dic_scn_var_comp'" + msg += f"\n\t Current CAM variable: {current_var}" + msg += f"\n\t Derived components for CAM variable {current_var}: {components}" + #adfobj.debug_log(msg) + Dic_comp={} + Dic_comp,missing_vars,needed_vars=SEbudget(adfobj,dic_SE,current_dir,current_files,components,ext1_SE) + + for comp in components: + # Write details to log file + msg += f"\n\t\t calculate derived component: {comp} for main variable, {current_var}" + adfobj.debug_log(msg) + + # Gather info for debugging + for var_m in missing_vars: + if var_m not in missing_vars_tot: + missing_vars_tot.append(var_m) + for var_n in needed_vars: + if var_n not in needed_vars_tot: + needed_vars_tot.append(var_n) + # End for + # End for + # Set dictionary for key of current variable with dictionary values of all + # necessary constituents for calculating the current variable + Dic_var_comp[current_var] = Dic_comp + Dic_scn_var_comp = Dic_var_comp + + # Critical threshholds, just run this once + # this is for finding tropospheric values + # Critical threshholds?\n", + # Just run this once\n", + tropospheric_method='NA' + try: + current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['Pressure'],ext1_SE) + Dic_crit=current_crit['Pressure'] + tropospheric_method='pressure' + msg += f"\n\t WARNING: Troposphere is defined as Pressure>500 hPa" + except: + current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['U'],ext1_SE) + Dic_crit=current_crit['U'] + Tropospheric=False + msg += f"\n\t WARNING: No way of defining troposphere was found in the model, budgets are total column" + # Log info to logging file + msg = f"chem/aerosol tables:" + msg += f"\n\t - potential missing variables from budget? {missing_vars_tot}" + adfobj.debug_log(msg) + + msg = f"chem/aerosol tables:" + msg += f"\n\t - needed variables for budget {needed_vars_tot}" + adfobj.debug_log(msg) + + return Dic_crit,Dic_scn_var_comp,Tropospheric,tropospheric_method +##### + + +def SEbudget(adfobj,dic_SE,data_dir,files,vars,ext1_SE,**kwargs): + """ + Function used for getting the data for the budget calculation. This is the + chunk of code that takes the longest by far. + + Example: + ** This is for both chemistry and aeorosl calculations + + dic_SE: dictionary specyfing what variables to get. For example, + for precipitation you can define SE as: + dic_SE['PRECT']={'PRECC'+ext1_SE:8.64e7,'PRECL'+ext1_SE:8.64e7} + - It means to sum the file variables "PRECC" and "PRECL" + for my arbitrary desired variable named "PRECT" + + - It also has the option to apply conversion factors. + For instance, PRECL and PRECC are in m/s. 8.64e7 is used to convernt m/s to mm/day + + + data_dir: string of the directory that contains the files. always end with '/' + + files: list of the files to be read + + var: string showing the variable to be extracted. + -> this will be the individual componnent, ie O3_CHMP, SOA_WDF, etc. + """ + + # gas constanct + Rgas=287.04 #[J/K/Kg]=8.314/0.028965 + + # Set lists to gather necessary variables for logging + missing_vars = [] + needed_vars = [] + Dic_all_data={} + +# all_data=[] + for file in range(len(files)): + + ds=xr.open_dataset(Path(data_dir) / files[file]) + + # Calculate these just once + if file==0: + mock_2d=np.zeros_like(np.array(ds['PS'+ext1_SE].isel(time=0))) + mock_3d=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) + + try: + delP=np.array(ds['PDELDRY'+ext1_SE].isel(time=0)) + except: + + hyai=np.array(ds['hyai']) + hybi=np.array(ds['hybi']) + + try: + PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) + except: + PS=np.array(ds['PS'+ext1_SE].isel(time=0)) + # End try/except + + P0=1e5 + Plevel=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) + + for i in range(len(Plevel)): + Plevel[i]=hyai[i]*P0+hybi[i]*PS + + delP=Plevel[1:]-Plevel[:-1] + + for var in vars: + if file == 0: + Dic_all_data[var]=[] + + + # Star gathering of variable data + + if var=='TROP_P': + data=np.array(ds['TROP_P'+ext1_SE].isel(time=0))/100 + elif var== 'Pressure': + try: + data=np.array(ds['PMID'+ext1_SE].isel(time=0))/100 + except: + hyam=np.array(ds['hyam']) + hybm=np.array(ds['hybm']) + try: + PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) + except: + PS=np.array(ds['PS'+ext1_SE].isel(time=0)) + P0=1e5 + data=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) + for i in range(len(data)): + data[i]=hyam[i]*P0+hybm[i]*PS + data=data/100 + else: + + + data=[] + for i in dic_SE[var].keys(): + + if file == 0: + msg = f"chem/aerosol tables: 'SEbudget'" + msg += f"\n\t\t ** variable(s) needed for derived var {var}: {dic_SE[var].keys()}" + msg += f"\n\t\t - constituent for derived var {var}: {i}" + adfobj.debug_log(msg) + if i not in needed_vars: + needed_vars.append(i) + if ((i!='PS'+ext1_SE) and (i!='U'+ext1_SE) ) : + data.append(np.array(ds[i].isel(time=0))*dic_SE[var][i]) + else: + if i=='PS'+ext1_SE: + data.append(mock_2d) + else: + data.append(mock_3d) + # End if + if file == 0: + + if var not in missing_vars: + if var!='U': # This is to avoid confusion between U variable or U mock! + missing_vars.append(var) + msg += f"\n\t\t - no variable was found for var {var}: {i}" + + # End if + + # Get total summed data for this history file data + data=np.sum(data,axis=0) + # End try/except + + if ('CHML' in var) or ('CHMP' in var) : + Temp=np.array(ds['T'+ext1_SE].isel(time=0)) + try: + Pres=np.array(ds['PMID'+ext1_SE].isel(time=0)) + except: + hyam=np.array(ds['hyam']) + hybm=np.array(ds['hybm']) + try: + PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) + except: + PS=np.array(ds['PS'+ext1_SE].isel(time=0)) + P0=1e5 + Pres=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) + for i in range(len(Pres)): + Pres[i]=hyam[i]*P0+hybm[i]*PS + rho= Pres/(Rgas*Temp) + data=data*delP/rho + elif ('BURDEN' in var): + data=data*delP + else: + data=data + # End if + # Add data to list + Dic_all_data[var].append(data) + ds.close() + for var in vars: # Take mean + Dic_all_data[var]=np.nanmean(Dic_all_data[var],axis=0) + + + return Dic_all_data,missing_vars,needed_vars +##### + + +def calc_budget_data(current_var, Dic_scn_var_comp, area, trop, inside, num_yrs, duration, AEROSOLS): + """ + Function to run through desired table values for calculations for the table entries + """ + + # Initialize full data dictionary for current table type + chem_dict = {} + + # Update variable marker if neccessary + if current_var == 'SO4': + specifier = ' S' + else: + specifier = '' + + # Calculate values for given variable + if 'AOD' in current_var: + # Burden + spc_burd = Dic_scn_var_comp[current_var][current_var+'_AOD'] + burden = np.ma.masked_where(inside==False,spc_burd) #convert Kg/m2 to Tg + BURDEN = np.ma.sum(burden*area)/np.ma.sum(area) + chem_dict[f"{current_var}_mean"] = np.round(BURDEN,5) + else: + # Surface Emissions + spc_sf = Dic_scn_var_comp[current_var][current_var+'_SF'] + tmp_sf = spc_sf + sf = np.ma.masked_where(inside==False,tmp_sf*area) #convert Kg/m2/s to Tg/yr + SF = np.ma.sum(sf*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_EMIS (Tg{specifier}/yr)"] = np.round(SF,5) + + # Elevated Emissions + spc_clxf = Dic_scn_var_comp[current_var][current_var+'_CLXF'] + tmp_clxf = spc_clxf + clxf = np.ma.masked_where(inside==False,tmp_clxf*area) #convert Kg/m2/s to Tg/yr + CLXF = np.ma.sum(clxf*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_EMIS_elevated (Tg{specifier}/yr)"] = np.round(CLXF,5) + + # Burden + spc_burd = Dic_scn_var_comp[current_var][current_var+'_BURDEN'] + spc_burd = np.where(np.isnan(trop),np.nan,spc_burd) + tmp_burden = np.nansum(spc_burd*area,axis=0) + burden = np.ma.masked_where(inside==False,tmp_burden) #convert Kg/m2 to Tg + BURDEN = np.ma.sum(burden*1e-9) + chem_dict[f"{current_var}_BURDEN (Tg{specifier})"] = np.round(BURDEN,5) + + # Chemical Loss + spc_chml = Dic_scn_var_comp[current_var][current_var+'_CHML'] + spc_chml = np.where(np.isnan(trop),np.nan,spc_chml) + tmp_chml = np.nansum(spc_chml*area,axis=0) + chml = np.ma.masked_where(inside==False,tmp_chml) #convert Kg/m2/s to Tg/yr + CHML = np.ma.sum(chml*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_CHEM_LOSS (Tg{specifier}/yr)"] = np.round(CHML,5) + + # Chemical Production + if current_var == 'SO4': # chemical production is basically the elevated emissions. + # We have removed it for SO4 budget. and put 0 here, so, we don't report it + chem_dict[f"{current_var}_CHEM_PROD (Tg{specifier}/yr)"] = 0 + else: + spc_chmp = Dic_scn_var_comp[current_var][current_var+'_CHMP'] + spc_chmp = np.where(np.isnan(trop),np.nan,spc_chmp) + tmp_chmp = np.nansum(spc_chmp*area,axis=0) + chmp = np.ma.masked_where(inside==False,tmp_chmp) #convert Kg/m2/s to Tg/yr + CHMP = np.ma.sum(chmp*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_CHEM_PROD (Tg{specifier}/yr)"] = np.round(CHMP,5) + # End if + + # Aerosol calculations + #--------------------- + if current_var in AEROSOLS: + + # Dry Deposition Flux + spc_ddfa = Dic_scn_var_comp[current_var][current_var+'_DDF'] + spc_ddfc = Dic_scn_var_comp[current_var][current_var+'_DDFC'] + spc_ddf = spc_ddfa +spc_ddfc + tmp_ddf = spc_ddf + ddf = np.ma.masked_where(inside==False,tmp_ddf*area) #convert Kg/m2/s to Tg/yr + DDF = np.ma.sum(ddf*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_DRYDEP (Tg{specifier}/yr)"] = np.round(DDF,5) + + # Wet deposition + spc_wdfa = Dic_scn_var_comp[current_var][current_var+'_WDF'] + spc_wdfc = Dic_scn_var_comp[current_var][current_var+'_WDFC'] + spc_wdf = spc_wdfa +spc_wdfc + tmp_wdf = spc_wdf + wdf = np.ma.masked_where(inside==False,tmp_wdf*area) #convert Kg/m2/s to Tg/yr + WDF = -1*np.ma.sum(wdf*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_WETDEP (Tg{specifier}/yr)"] = np.round(WDF,5) + + if current_var in ["SOA",'SO4']: + # gas-aerosol Exchange + spc_gaex = Dic_scn_var_comp[current_var][current_var+'_GAEX'] + tmp_gaex = spc_gaex + gaex = np.ma.masked_where(inside==False,tmp_gaex*area) #convert Kg/m2/s to Tg/yr + GAEX = np.ma.sum(gaex*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_GAEX (Tg{specifier}/yr)"] = np.round(GAEX,5) + + # LifeTime = Burden/(loss+deposition) no chemical loss for aerosols + LT = BURDEN/(DDF+WDF)* duration/86400/num_yrs # days + chem_dict[f"{current_var}_LIFETIME (days)"] = np.round(LT,5) + + if current_var == 'SO4': + # Aqueous Chemistry + spc_aqs = Dic_scn_var_comp[current_var][current_var+'_AQS'] + tmp_aqs = spc_aqs + aqs = np.ma.masked_where(inside==False,tmp_aqs*area) #convert Kg/m2/s to Tg/yr + AQS = np.ma.sum(aqs*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_AQUEOUS (Tg{specifier}/yr)"] = np.round(AQS,5) + + # Nucleation + spc_nucl = Dic_scn_var_comp[current_var][current_var+'_NUCL'] + tmp_nucl = spc_nucl + nucl = np.ma.masked_where(inside==False,tmp_nucl*area) #convert Kg/m2/s to Tg/yr + NUCL = np.ma.sum(nucl*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_NUCLEATION (Tg{specifier}/yr)"] = np.round(NUCL,5) + + # Gaseous calculations + #--------------------- + else: + # Dry Deposition Flux + spc_ddf = Dic_scn_var_comp[current_var][current_var+'_DDF'] + tmp_ddf = spc_ddf + ddf = np.ma.masked_where(inside==False,tmp_ddf*area) #convert Kg/m2/s to Tg/yr + DDF = np.ma.sum(ddf*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_DRYDEP (Tg/yr)"] = np.round(DDF,5) + + # Wet Deposition Flux + spc_wdf = Dic_scn_var_comp[current_var][current_var+'_WDF'] + tmp_wdf = spc_wdf + wdf = np.ma.masked_where(inside==False,tmp_wdf*area) #convert Kg/m2/s to Tg/yr + WDF = -1*np.ma.sum(wdf*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_WETDEP (Tg/yr)"] = np.round(WDF,5) + + # Total Deposition + TDEP = DDF + WDF + chem_dict[f"{current_var}_TDEP (Tg/yr)"] = np.round(TDEP,5) + + # LifeTime = Burden/(loss+deposition) + if current_var == "CH4": + LT = BURDEN/(CHML+DDF+WDF) # years + chem_dict[f"{current_var}_LIFETIME (years)"] = np.round(LT,5) + else: + if (CHML+DDF+WDF) > 0: + if CHML != 0: + LT = BURDEN/(CHML+DDF+WDF)*duration/86400/num_yrs # days + chem_dict[f"{current_var}_LIFETIME (days)"] = np.round(LT,5) + else: + # do not report lifetime if chemical loss (for gases) is not included in the model outputs + # and put 0 here, so, we don't report it + chem_dict[f"{current_var}_LIFETIME (days)"] = 0 + # End if + # End if + # End if + + #NET = CHMP-CHML + # Chemical Tendency + spc_tnd = Dic_scn_var_comp[current_var][current_var+'_TEND'] + spc_tnd = np.where(np.isnan(trop),np.nan,spc_tnd) + tmp_tnd = np.nansum(spc_tnd,axis=0) + tnd = np.ma.masked_where(inside==False,tmp_tnd) #convert Kg/s to Tg/yr + TND = np.ma.sum(tnd*duration*1e-9)/num_yrs + chem_dict[f"{current_var}_TEND (Tg/yr)"] = np.round(TND,5) + + # O3 dependent calculations + if current_var == "O3": + # Stratospheric-Tropospheric Exchange + STE = DDF-TND + chem_dict[f"{current_var}_STE (Tg/yr)"] = np.round(STE,5) + + # Lightning NOX production + spc_lno = Dic_scn_var_comp[current_var][current_var+'_LNO'] + tmp_lno = np.ma.masked_where(inside==False,spc_lno) + LNO = np.ma.sum(tmp_lno) + chem_dict[f"{current_var}_LNO (Tg N/yr)"] = np.round(LNO,5) + # End if (aerosol or gas) + return chem_dict +##### + + +def make_table(adfobj, vars, chem_type, Dic_scn_var_comp, areas, trops, case_names, nicknames, durations, insides, num_yrs, AEROSOLS): + """ + Create CSV table for aeorosols and gases, if applicable + + Table includes column values of variable, case(s), difference (if applicable) + + If this is a single model vs model run: 4 columns + first column: variables names, + second column: test case variable values + third column: baseline case variable values + final column: difference of test and baseline. + If this is a model vs obs run: 2 columns + first column: variables names, + second column: test case variable values + """ + # Initialize an empty dictionary to store DataFrames + dfs = {} + + #Special ADF variable which contains the output paths for + #all generated plots and tables for each case: + output_locs = adfobj.plot_location + + #Convert output location string to a Path object: + output_location = Path(output_locs[0]) + + # Loop over model cases + + for i,case in enumerate(nicknames): + + nickname = case + + # Collect row data in a list of dictionaries + #durations[case] + rows = [] + for current_var in vars: + chem_dict = calc_budget_data(current_var, Dic_scn_var_comp[case], areas[case], trops[case], insides[case], + num_yrs[case], durations[case], AEROSOLS) + # Loop through table variables + for key, val in chem_dict.items(): + if val != 0: # Skip variables with a value of 0 + print(f"\t - Variable '{key}' being added to table") + rows.append({'variable': key, nickname: np.round(val, 3)}) + elif 'OASISS_EMIS (' in key: # the paranthesis is to ignore EMIS_Elevated variables! + print(f"\t - Variable '{key}' being added to table") + rows.append({'variable': key, nickname: np.round(val, 3)}) + else: + msg = f"chem/aerosol tables:" + msg += f"\n\t - Variable '{key}' has value of 0, will not add to table" + adfobj.debug_log(msg) + # End if + # End for + # End for + + # Create the DataFrame for the current case + table_df = pd.DataFrame(rows) + + if chem_type == 'gases': + # Replace compound names directly in the DataFrame + replacements = { + 'MTERP': 'Monoterpene', + 'CH3OH': 'Methanol', + 'CH3COCH3': 'Acetone', + 'O3_LNO': 'LNOx_PROD' + } + table_df['variable'] = table_df['variable'].replace(replacements, regex=True) + # End if + + # Store the DataFrame in the dictionary + dfs[nickname] = table_df + + # End for + + # Merge the DataFrames on the 'variable' column + if len(case_names) == 2: + + table_df = pd.merge(dfs[nicknames[0]], dfs[nicknames[1]], on='variable') + + # Calculate the differences between case columns + table_df['difference'] = table_df[nicknames[0]] - table_df[nicknames[1]] + + #Create output file name: + output_csv_file = output_location / f'ADF_amwg_{chem_type}_table.csv' + # Save table to CSV and add table dataframe to website (if enabled) + table_df.to_csv(output_csv_file, index=False) + #adfobj.add_website_data(table_df, chem_type, case, plot_type="Tables") + adfobj.add_website_data(table_df, chem_type, case_names[0], plot_type="Tables") + +##### From 1297f5ed3e01b56aa0e794af22121eb9b3d0e4f6 Mon Sep 17 00:00:00 2001 From: Behrooz-Roozitalab Date: Wed, 3 Sep 2025 13:48:22 -0600 Subject: [PATCH 52/91] update tropopause definition --- config_DCOTSS.yaml | 321 ---- config_WACCM_beta06_WACCM_FWHIST.yaml | 583 ------- config_amwg_default_plots.yaml | 470 ------ config_cam_baseline_example.yaml | 534 ------- config_cam_baseline_example_testENSO.yaml | 537 ------- config_for_Simone_beta05.yaml | 474 ------ .../aerosol_gas_tables_Tropopause_version0.py | 1390 ----------------- .../aerosol_gas_tables_Tropopause_version1.py | 1379 ---------------- 8 files changed, 5688 deletions(-) delete mode 100644 config_DCOTSS.yaml delete mode 100644 config_WACCM_beta06_WACCM_FWHIST.yaml delete mode 100644 config_amwg_default_plots.yaml delete mode 100644 config_cam_baseline_example.yaml delete mode 100644 config_cam_baseline_example_testENSO.yaml delete mode 100644 config_for_Simone_beta05.yaml delete mode 100644 scripts/analysis/aerosol_gas_tables_Tropopause_version0.py delete mode 100644 scripts/analysis/aerosol_gas_tables_Tropopause_version1.py diff --git a/config_DCOTSS.yaml b/config_DCOTSS.yaml deleted file mode 100644 index 0d8913f0a..000000000 --- a/config_DCOTSS.yaml +++ /dev/null @@ -1,321 +0,0 @@ -#============================== -#config_cam_baseline_example.yaml - -#This is the main CAM diagnostics config file -#for doing comparisons of a CAM run against -#another CAM run, or a CAM baseline simulation. - -#Currently, if one is on NCAR's Casper or -#Cheyenne machine, then only the diagnostic output -#paths are needed, at least to perform a quick test -#run (these are indicated with "MUST EDIT" comments). -#Running these diagnostics on a different machine, -#or with a different, non-example simulation, will -#require additional modifications. -# -#Config file Keywords: -#-------------------- -# -#1. Using ${xxx} will substitute that text with the -# variable referenced by xxx. For example: -# -# cam_case_name: cool_run -# cam_climo_loc: /some/where/${cam_case_name} -# -# will set "cam_climo_loc" in the diagnostics package to: -# /some/where/cool_run -# -# Please note that currently this will only work if the -# variable only exists in one location in the file. -# -#2. Using ${.xxx} will do the same as -# keyword 1 above, but specifies which sub-section the -# variable is coming from, which is necessary for variables -# that are repeated in different subsections. For example: -# -# diag_basic_info: -# cam_climo_loc: /some/where/${diag_cam_climo.start_year} -# -# diag_cam_climo: -# start_year: 1850 -# -# will set "cam_climo_loc" in the diagnostics package to: -# /some/where/1850 -# -#Finally, please note that for both 1 and 2 the keywords must be lowercase. -#This is because future developments will hopefully use other keywords -#that are uppercase. Also please avoid using periods (".") in variable -#names, as this will likely cause issues with the current file parsing -#system. -#-------------------- -# -##============================== -# -# This file doesn't (yet) read environment variables, so the user must -# set this themselves. It is also a good idea to search the doc for 'user' -# to see what default paths are being set for output/working files. -# -# Note that the string 'USER-NAME-NOT-SET' is used in the jupyter script -# to check for a failure to customize -# -user: 'behroozr' - - -#This first set of variables specify basic info used by all diagnostic runs: -diag_basic_info: - - #Is this a model vs observations comparison? - #If "false" or missing, then a model-model comparison is assumed: - compare_obs: false - - #Generate HTML website (assumed false if missing): - #Note: The website files themselves will be located in the path - #specified by "cam_diag_plot_loc", under the "/website" subdirectory, - #where "" is the subdirectory created for this particular diagnostics run - #(usually "case_vs_obs_XXX" or "case_vs_baseline_XXX"). - create_html: false - - #Location of observational datasets: - #Note: this only matters if "compare_obs" is true and the path - #isn't specified in the variable defaults file. - obs_data_loc: /glade/campaign/cgd/amp/amwg/ADF_obs - - #Location where re-gridded and interpolated CAM climatology files are stored: - cam_regrid_loc: /glade/derecho/scratch/${user}/ADF/regrid - - #Overwrite CAM re-gridded files? - #If false, or missing, then regridding will be skipped for regridded variables - #that already exist in "cam_regrid_loc": - cam_overwrite_regrid: false - - #Location where diagnostic plots are stored: - cam_diag_plot_loc: /glade/derecho/scratch/${user}/ADF/plots - - #Location of ADF variable plotting defaults YAML file: - #If left blank or missing, ADF/lib/adf_variable_defaults.yaml will be used - #Uncomment and change path for custom variable defaults file - #defaults_file: /some/path/to/defaults/file.yaml - - #Vertical pressure levels (in hPa) on which to plot 3-D variables - #when using horizontal (e.g. lat/lon) map projections. - #If this config option is missing, then no 3-D variables will be plotted on - #horizontal maps. Please note too that pressure levels must currently match - #what is available in the observations file in order to be plotted in a - #model vs obs run: - plot_press_levels: [200,850] - - #Longitude line on which to center all lat/lon maps. - #If this config option is missing then the central - #longitude will default to 180 degrees E. - central_longitude: 180 - - #Number of processors on which to run the ADF. - #If this config variable isn't present then - #the ADF defaults to one processor. Also, if - #you set it to "*" then it will default - #to all of the processors available on a - #single node/machine: - num_procs: 8 - - #If set to true, then redo all plots even if they already exist. - #If set to false, then if a plot is found it will be skipped: - redo_plot: false - -#This second set of variables provides info for the CAM simulation(s) being diagnosed: -diag_cam_climo: - - # History file list of strings to match - # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] - # Only affects timeseries as everything else uses the created timeseries - # Default: - hist_str: cam.hm - - #Calculate climatologies? - #If false, the climatology files will not be created: - calc_cam_climo: false - - #Overwrite CAM climatology files? - #If false, or not prsent, then already existing climatology files will be skipped: - cam_overwrite_climo: false - - #Name of CAM case (or CAM run name): - cam_case_name: f.e22.FCnudged.f09_32L.slh_released.2019.DCOTSS_ACCLIP.finn.cams6Mosaic.v01 - - #Case nickname - #NOTE: if nickname starts with '0' - nickname must be in quotes! - # ie '026a' as opposed to 026a - #If missing or left blank, will default to cam_case_name - case_nickname: DCOTSS_1deg_2022 #cool nickname - - #Location of CAM history (h0) files: - #Example test files - cam_hist_loc: /glade/campaign/acom/acom-weather/behroozr/NASA_DCOTSS/cases2024/${diag_cam_climo.cam_case_name}/atm/hist/ - - #Location of CAM climatologies (to be created and then used by this script) - cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/climo - - #model year when time series files should start: - #Note: Leaving this entry blank will make time series - # start at earliest available year. - start_year: 2022 #10 - - #model year when time series files should end: - #Note: Leaving this entry blank will make time series - # end at latest available year. - end_year: 2022 #14 - - #Do time series files exist? - #If True, then diagnostics assumes that model files are already time series. - #If False, or if simply not present, then diagnostics will attempt to create - #time series files from history (time-slice) files: - cam_ts_done: false - - #Save interim time series files? - #WARNING: This can take up a significant amount of space, - # but will save processing time the next time - cam_ts_save: false - - #Overwrite time series files, if found? - #If set to false, then time series creation will be skipped if files are found: - cam_overwrite_ts: false - - #Location where time series files are (or will be) stored: - cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/ts - - - -#This third set of variables provide info for the CAM baseline climatologies. -#This only matters if "compare_obs" is false: -diag_cam_baseline_climo: - - # History file list of strings to match - # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] - # Only affects timeseries as everything else uses the created timeseries - # Default: - hist_str: cam.h0 - - #Calculate cam baseline climatologies? - #If false, the climatology files will not be created: - calc_cam_climo: false - - #Overwrite CAM climatology files? - #If false, or not present, then already existing climatology files will be skipped: - cam_overwrite_climo: false - - #Name of CAM baseline case: - cam_case_name: f.e22.FCnudged.f09_32L.slh_released.2019.DCOTSS_ACCLIP.finn.cams6Mosaic.v01 - - #Baseline case nickname - #NOTE: if nickname starts with '0' - nickname must be in quotes! - # ie '026a' as opposed to 026a - #If missing or left blank, will default to cam_case_name - case_nickname: DCOTSS_1deg_2021 #cool nickname - - #Location of CAM baseline history (h0) files: - #Example test files - cam_hist_loc: /glade/campaign/acom/acom-weather/behroozr/NASA_DCOTSS/cases2024/${diag_cam_baseline_climo.cam_case_name}/atm/hist/ - - #Location of baseline CAM climatologies: - cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/climo - - #model year when time series files should start: - #Note: Leaving this entry blank will make time series - # start at earliest available year. - start_year: 2021 #10 - - #model year when time series files should end: - #Note: Leaving this entry blank will make time series - # end at latest available year. - end_year: 2021 #14 - - #Do time series files need to be generated? - #If True, then diagnostics assumes that model files are already time series. - #If False, or if simply not present, then diagnostics will attempt to create - #time series files from history (time-slice) files: - cam_ts_done: false - - #Save interim time series files for baseline run? - #WARNING: This can take up a significant amount of space: - cam_ts_save: false - - #Overwrite baseline time series files, if found? - #If set to false, then time series creation will be skipped if files are found: - cam_overwrite_ts: false - - #Location where time series files are (or will be) stored: - cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/ts - - - -#+++++++++++++++++++++++++++++++++++++++++++++++++++ -#These variables below only matter if you are using -#a non-standard method, or are adding your own -#diagnostic scripts. -#+++++++++++++++++++++++++++++++++++++++++++++++++++ - -#Note: If you want to pass arguments to a particular script, you can -#do it like so (using the "averaging_example" script in this case): -# - {create_climo_files: {kwargs: {clobber: true}}} - -#Name of time-averaging scripts being used to generate climatologies. -#These scripts must be located in "scripts/averaging": -time_averaging_scripts: - - create_climo_files - -#Name of regridding scripts being used. -#These scripts must be located in "scripts/regridding": -regridding_scripts: - - regrid_and_vert_interp - -#List of analysis scripts being used. -#These scripts must be located in "scripts/analysis": -analysis_scripts: - - aerosol_gas_tables - -#List of plotting scripts being used. -#These scripts must be located in "scripts/plotting": -plotting_scripts: - # - global_latlon_map - # - global_latlon_vect_map - # - zonal_mean - # - meridional_mean - # - ozone_diagnostics - -#List of CAM variables that will be processesd: -#If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed -diag_var_list: - - CO - #- tr_CH2CL2_AS - # - CH2CL2 - # - O3 - #- AODVISdn - #- BC - - # - SWCF - #- LWCF - #- PRECC - #- PRECL - #- PSL - #- Q - #- U - #- T - #- RELHUM - #- TREFHT - #- TS - #- TAUX - #- TAUY - #- FSNT - #- FLNT - #- RESTOM - # - AODVISdn - #- Q - #- BC - #- POM - #- SO4 - #- SOA - #- DUST - #- SeaSalt - #- O3 - - -#END OF FILE diff --git a/config_WACCM_beta06_WACCM_FWHIST.yaml b/config_WACCM_beta06_WACCM_FWHIST.yaml deleted file mode 100644 index 722c669f6..000000000 --- a/config_WACCM_beta06_WACCM_FWHIST.yaml +++ /dev/null @@ -1,583 +0,0 @@ -#============================== -#config_cam_baseline_example.yaml - -#This is the main CAM diagnostics config file -#for doing comparisons of a CAM run against -#another CAM run, or a CAM baseline simulation. - -#Currently, if one is on NCAR's Casper or -#Cheyenne machine, then only the diagnostic output -#paths are needed, at least to perform a quick test -#run (these are indicated with "MUST EDIT" comments). -#Running these diagnostics on a different machine, -#or with a different, non-example simulation, will -#require additional modifications. -# -#Config file Keywords: -#-------------------- -# -#1. Using ${xxx} will substitute that text with the -# variable referenced by xxx. For example: -# -# cam_case_name: cool_run -# cam_climo_loc: /some/where/${cam_case_name} -# -# will set "cam_climo_loc" in the diagnostics package to: -# /some/where/cool_run -# -# Please note that currently this will only work if the -# variable only exists in one location in the file. -# -#2. Using ${.xxx} will do the same as -# keyword 1 above, but specifies which sub-section the -# variable is coming from, which is necessary for variables -# that are repeated in different subsections. For example: -# -# diag_basic_info: -# cam_climo_loc: /some/where/${diag_cam_climo.start_year} -# -# diag_cam_climo: -# start_year: 1850 -# -# will set "cam_climo_loc" in the diagnostics package to: -# /some/where/1850 -# -#Finally, please note that for both 1 and 2 the keywords must be lowercase. -#This is because future developments will hopefully use other keywords -#that are uppercase. Also please avoid using periods (".") in variable -#names, as this will likely cause issues with the current file parsing -#system. -#-------------------- -# -##============================== -# -# This file doesn't (yet) read environment variables, so the user must -# set this themselves. It is also a good idea to search the doc for 'user' -# to see what default paths are being set for output/working files. -# -# Note that the string 'USER-NAME-NOT-SET' is used in the jupyter script -# to check for a failure to customize -# -user: 'behroozr' - - -#This first set of variables specify basic info used by all diagnostic runs: -diag_basic_info: - - #Is this a model vs observations comparison? - #If "false" or missing, then a model-model comparison is assumed: - compare_obs: false - - #Generate HTML website (assumed false if missing): - #Note: The website files themselves will be located in the path - #specified by "cam_diag_plot_loc", under the "/website" subdirectory, - #where "" is the subdirectory created for this particular diagnostics run - #(usually "case_vs_obs_XXX" or "case_vs_baseline_XXX"). - create_html: true - - #Location of observational datasets: - #Note: this only matters if "compare_obs" is true and the path - #isn't specified in the variable defaults file. - obs_data_loc: /glade/campaign/cgd/amp/amwg/ADF_obs - - #Location where re-gridded and interpolated CAM climatology files are stored: - cam_regrid_loc: /glade/derecho/scratch/${user}/ADF/regrid - - #Overwrite CAM re-gridded files? - #If false, or missing, then regridding will be skipped for regridded variables - #that already exist in "cam_regrid_loc": - cam_overwrite_regrid: false - - #Location where diagnostic plots are stored: - cam_diag_plot_loc: /glade/derecho/scratch/${user}/ADF/plots - - #Location of ADF variable plotting defaults YAML file: - #If left blank or missing, ADF/lib/adf_variable_defaults.yaml will be used - #Uncomment and change path for custom variable defaults file - #defaults_file: /some/path/to/defaults/file.yaml - - #Vertical pressure levels (in hPa) on which to plot 3-D variables - #when using horizontal (e.g. lat/lon) map projections. - #If this config option is missing, then no 3-D variables will be plotted on - #horizontal maps. Please note too that pressure levels must currently match - #what is available in the observations file in order to be plotted in a - #model vs obs run: - plot_press_levels: [200,850] - - #Longitude line on which to center all lat/lon maps. - #If this config option is missing then the central - #longitude will default to 180 degrees E. - central_longitude: 180 - - #Number of processors on which to run the ADF. - #If this config variable isn't present then - #the ADF defaults to one processor. Also, if - #you set it to "*" then it will default - #to all of the processors available on a - #single node/machine: - num_procs: 8 - - #If set to true, then redo all plots even if they already exist. - #If set to false, then if a plot is found it will be skipped: - redo_plot: false - -#This second set of variables provides info for the CAM simulation(s) being diagnosed: -diag_cam_climo: - - # History file list of strings to match - # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] - # Only affects timeseries as everything else uses the created timeseries - # Default: - hist_str: cam.h0a - #hist_str: cam.hm - - #Calculate climatologies? - #If false, the climatology files will not be created: - calc_cam_climo: false - - #Overwrite CAM climatology files? - #If false, or not prsent, then already existing climatology files will be skipped: - cam_overwrite_climo: false - - #Name of CAM case (or CAM run name): - cam_case_name: f.e30_beta06_megan.FWHIST_f09_f09_mg17v1.L70.cam6.clm6.002 - - #Case nickname - #NOTE: if nickname starts with '0' - nickname must be in quotes! - # ie '026a' as opposed to 026a - #If missing or left blank, will default to cam_case_name - case_nickname: "WACCM (beta06)" - - #Location of CAM history (h0) files: - #Example test files - cam_hist_loc: /glade/derecho/scratch/shawnh/archive/f.e30_beta06_megan.FWHIST_f09_f09_mg17v1.L70.cam6.clm6.002/atm/hist - #cam_hist_loc: /glade/derecho/scratch/behroozr/Budgets - - #Location of CAM climatologies (to be created and then used by this script) - cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/climo - - #model year when time series files should start: - #Note: Leaving this entry blank will make time series - # start at earliest available year. - start_year: 2010 - - #model year when time series files should end: - #Note: Leaving this entry blank will make time series - # end at latest available year. - end_year: 2010 - - #Do time series files exist? - #If True, then diagnostics assumes that model files are already time series. - #If False, or if simply not present, then diagnostics will attempt to create - #time series files from history (time-slice) files: - cam_ts_done: false - - #Save interim time series files? - #WARNING: This can take up a significant amount of space, - # but will save processing time the next time - cam_ts_save: true - - #Overwrite time series files, if found? - #If set to false, then time series creation will be skipped if files are found: - cam_overwrite_ts: false - - #Location where time series files are (or will be) stored: - cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/ts - - #TEM diagnostics - #--------------- - #TEM history file number - #If missing or blank, ADF will default to h4 - tem_hist_str: cam.h4a - - #Location where TEM files are stored: - #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! - cam_tem_loc: /glade/derecho/scratch/${user}/${diag_cam_climo.cam_case_name}/tem/ - - #Overwrite TEM files, if found? - #If set to false, then TEM creation will be skipped if files are found: - overwrite_tem: false - - #---------------------- - - #You can alternatively provide a list of cases, which will make the ADF - #apply the same diagnostics to each case separately in a single ADF session. - #All of the config variables below show how it is done, and are the only ones - #that need to be lists. This also automatically enables the generation of - #a "main_website" in "cam_diag_plot_loc" that brings all of the different cases - #together under a single website. - - #Also please note that config keywords cannot currently be used in list mode. - - #cam_case_name: - # - b.e23_alpha17f.BLT1850.ne30_t232.098 - # - b.e23_alpha17f.BLT1850.ne30_t232.095 - - #Case nickname - #NOTE: if nickname starts with '0' - nickname must be in quotes! - # ie '026a' as opposed to 026a - #If missing or left blank, will default to cam_case_name - #case_nickname: - # - cool nickname - # - cool nickname 2 - - #calc_cam_climo: - # - true - # - true - - #cam_overwrite_climo: - # - false - # - false - - #cam_hist_loc: - # - /glade/campaign/cgd/amp/amwg/ADF_test_cases/b.e23_alpha17f.BLT1850.ne30_t232.098 - # - /glade/campaign/cgd/amp/amwg/ADF_test_cases/b.e23_alpha17f.BLT1850.ne30_t232.095 - - #cam_climo_loc: - # - /some/where/you/want/to/have/climo_files/ #MUST EDIT! - # - /the/same/or/some/other/climo/files/location - - #start_year: - # - 10 - # - 10 - - #end_year: - # - 14 - # - 14 - - #cam_ts_done: - # - false - # - false - - #cam_ts_save: - # - true - # - true - - #cam_overwrite_ts: - # - false - # - false - - #cam_ts_loc: - # - /some/where/you/want/to/have/time_series_files - # - /same/or/different/place/you/want/files - - #TEM diagnostics - #--------------- - #TEM history file number - #If missing or blank, ADF will default to h4 - #tem_hist_str: - # - cam.h4 - # - cam.h# - - #Location where TEM files are stored: - #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! - #cam_tem_loc: - # - /some/where/you/want/to/have/TEM_files/ - # - /same/or/different/place/you/want/TEM_files/ - - #Overwrite TEM files, if found? - #If set to false, then TEM creation will be skipped if files are found: - #overwrite_tem: - # - false - # - true - - #---------------------- - - -#This third set of variables provide info for the CAM baseline climatologies. -#This only matters if "compare_obs" is false: -diag_cam_baseline_climo: - - # History file list of strings to match - # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] - # Only affects timeseries as everything else uses the created timeseries - # Default: - hist_str: cam.h0 - - #Calculate cam baseline climatologies? - #If false, the climatology files will not be created: - calc_cam_climo: false - - #Overwrite CAM climatology files? - #If false, or not present, then already existing climatology files will be skipped: - cam_overwrite_climo: false - - #Name of CAM baseline case: - cam_case_name: f.e22.FWHISTnudged.f09_f09.cesm2.2.0.2001-2021.001 - - #Baseline case nickname - #NOTE: if nickname starts with '0' - nickname must be in quotes! - # ie '026a' as opposed to 026a - #If missing or left blank, will default to cam_case_name - case_nickname: "WACCM (DOUGK)" #cool nickname - - #Location of CAM baseline history (h0) files: - #Example test files - cam_hist_loc: /glade/campaign/acom/acom-climate/UTLS/shawnh/archive/f.e22.FWHISTnudged.f09_f09.cesm2.2.0.2001-2021.001/atm/hist - - #Location of baseline CAM climatologies: - cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/climo - - #model year when time series files should start: - #Note: Leaving this entry blank will make time series - # start at earliest available year. - start_year: 2010 - - #model year when time series files should end: - #Note: Leaving this entry blank will make time series - # end at latest available year. - end_year: 2010 - - #Do time series files need to be generated? - #If True, then diagnostics assumes that model files are already time series. - #If False, or if simply not present, then diagnostics will attempt to create - #time series files from history (time-slice) files: - cam_ts_done: false - - #Save interim time series files for baseline run? - #WARNING: This can take up a significant amount of space: - cam_ts_save: true - - #Overwrite baseline time series files, if found? - #If set to false, then time series creation will be skipped if files are found: - cam_overwrite_ts: false - - #Location where time series files are (or will be) stored: - cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/ts - - #TEM diagnostics - #--------------- - #TEM history file number - #If missing or blank, ADF will default to h4 - tem_hist_str: cam.h4a - - #Location where TEM files are stored: - #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! - cam_tem_loc: /glade/derecho/scratch/${user}/${diag_cam_baseline_climo.cam_case_name}/tem/ - - #Overwrite TEM files, if found? - #If set to false, then TEM creation will be skipped if files are found: - overwrite_tem: false - - -#This fourth set of variables provides settings for calling the Climate Variability -# Diagnostics Package (CVDP). If cvdp_run is set to true the CVDP will be set up and -# run in background mode, likely completing after the ADF has completed. -# If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed -# in the diag_var_list variable listing. -# For more CVDP information: https://www.cesm.ucar.edu/working_groups/CVC/cvdp/ -diag_cvdp_info: - - # Run the CVDP on the listed run(s)? - cvdp_run: false - - # CVDP code path, sets the location of the CVDP codebase - # CGD systems path = /home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - # CISL systems path = /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - # github location = https://github.com/NCAR/CVDP-ncl - cvdp_codebase_loc: /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - - # Location where cvdp codebase will be copied to and diagnostic plots will be stored - cvdp_loc: /glade/derecho/scratch/${user}/ADF/cvdp/ - - # tar up CVDP results? - cvdp_tar: false - -# This set of variables provides settings for calling NOAA's -# Model Diagnostic Task Force (MDTF) diagnostic package. -# https://github.com/NOAA-GFDL/MDTF-diagnostics -# -# If mdtf_run: true, the MDTF will be set up and -# run in background mode, likely completing after the ADF has completed. -# -# WARNING: This currently only runs on CASPER (not derecho) -# -# The variables required depend on the diagnostics (PODs) selected. -# AMWG-developed PODS and their required variables: -# (Note that PRECT can be computed from PRECC & PRECL) -# - MJO_suite: daily PRECT, FLUT, U850, U200, V200 (all required) -# - Wheeler-Kiladis Wavenumber Frequency Spectra: daily PRECT, FLUT, U200, U850, OMEGA500 -# (will use what is available) -# - Blocking (Rich Neale): daily OMEGA500 -# - Precip Diurnal Cycle (Rich Neale): 3-hrly PRECT -# -# Many other diagnostics are available; see -# https://mdtf-diagnostics.readthedocs.io/en/main/sphinx/start_overview.html - -# -diag_mdtf_info: - # Run the MDTF on the model cases - mdtf_run: false - - # The file that will be written by ADF to input to MDTF. Call this whatever you want. - mdtf_input_settings_filename : mdtf_input.json - - ## MDTF code path, sets the location of the MDTF codebase and pre-compiled conda envs - # CHANGE if you have any: your own MDTF code, installed conda envs and/or obs_data - - mdtf_codebase_path : /glade/campaign/cgd/amp/amwg/mdtf - mdtf_codebase_loc : ${mdtf_codebase_path}/MDTF-diagnostics.v3.1.20230817.ADF - conda_root : /glade/u/apps/opt/conda - conda_env_root : ${mdtf_codebase_path}/miniconda2/envs.MDTFv3.1.20230412/ - OBS_DATA_ROOT : ${mdtf_codebase_path}/obs_data - - # SET this to a writable dir. The ADF will place ts files here for the MDTF to read (adds the casename) - MODEL_DATA_ROOT : ${diag_cam_climo.cam_ts_loc}/mdtf/inputdata/model - - # Choose diagnostics (PODs). Full list of available PODs: https://github.com/NOAA-GFDL/MDTF-diagnostics - pod_list : [ "MJO_suite" ] - - # Intermediate/output file settings - make_variab_tar: false # tar up MDTF results - save_ps : false # save postscript figures in addition to bitmaps - save_nc : false # save netCDF files of processed data (recommend true when starting with new model data) - overwrite: true # overwrite results in OUTPUT_DIR; otherwise results will be saved under a unique name - - # Settings used in debugging: - verbose : 3 # Log verbosity level. - test_mode: false # Set to true for framework test. Data is fetched but PODs are not run. - dry_run : false # Framework test. No external commands are run and no remote data is copied. Implies test_mode. - - # Settings that shouldn't change in ADF implementation for now - data_type : single_run # single_run or multi_run (only works with single right now) - data_manager : Local_File # Fetch data or it is local? - environment_manager : Conda # Manage dependencies - - - -#+++++++++++++++++++++++++++++++++++++++++++++++++++ -#These variables below only matter if you are using -#a non-standard method, or are adding your own -#diagnostic scripts. -#+++++++++++++++++++++++++++++++++++++++++++++++++++ - -#Note: If you want to pass arguments to a particular script, you can -#do it like so (using the "averaging_example" script in this case): -# - {create_climo_files: {kwargs: {clobber: true}}} - -#Name of time-averaging scripts being used to generate climatologies. -#These scripts must be located in "scripts/averaging": -#time_averaging_scripts: -# - create_climo_files - #- create_TEM_files #To generate TEM files, please un-comment - -#Name of regridding scripts being used. -#These scripts must be located in "scripts/regridding": -#regridding_scripts: -# - regrid_and_vert_interp - -#List of analysis scripts being used. -#These scripts must be located in "scripts/analysis": -analysis_scripts: - # - amwg_table - - aerosol_gas_tables - -#List of plotting scripts being used. -#These scripts must be located in "scripts/plotting": -#plotting_scripts: -# - global_latlon_map -# - global_latlon_vect_map -# - zonal_mean -# - meridional_mean -# - polar_map -# - cam_taylor_diagram -# - qbo -# - ozone_diagnostics -# - tape_recorder - #- MOPITT -# - seasonal_cycle - #- tem - #- regional_map_multicase #To use this please un-comment and fill-out - #the "region_multicase" section below - -#List of CAM variables that will be processesd: -#If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed -diag_var_list: - - SWCF -# - LWCF -# - PRECC -# - PRECL -# - PSL -# - Q -# - U -# - T -# - RELHUM -# - TREFHT -# - TS -# - TAUX -# - TAUY -# - FSNT -# - FLNT -# - LANDFRAC -# - O3 -# - O3S -# - CO -# - CO2 -# - H2O -# - NOX -# - NOY -# - CLDICE -# - DMS -# - EXTINCTdn -# - CFC11 -# - N2O -# - HNO3 -# - ISOP -# - CH4 -# - OH -# - SAD_TROP -# - SAD_AERO -# - SAD_SULFC -# - LNO_PROD -# - bc_a1 -# - bc_a4 -# - SO2 -# - dst_a1 -# - dst_a2 -# - dst_a3 -# - ncl_a1 -# - ncl_a2 -# - ncl_a3 -# - num_a1 -# - num_a2 -# - num_a3 -# - num_a4 -# - num_a5 -# - pom_a1 -# - pom_a4 -# - so4_a1 -# - so4_a2 -# - so4_a3 -# - so4_a5 -# - FLASHFRQ -# - LNO_COL_PROD -## - AODDUST -# - AODVIS -# - AODVISdn -# - MEG_ISOP - -# -# MDTF recommended variables -# - FLUT -# - OMEGA500 -# - PRECT -# - PS -# - PSL -# - U200 -# - U850 -# - V200 -# - V850 - -# Options for multi-case regional contour plots (./plotting/regional_map_multicase.py) -# region_multicase: -# region_spec: [slat, nlat, wlon, elon] -# region_time_option: # If calendar, will look for specified years. If zeroanchor will use a nyears starting from year_offset from the beginning of timeseries -# region_start_year: -# region_end_year: -# region_nyear: -# region_year_offset: -# region_month: -# region_season: -# region_variables: - -#END OF FILE diff --git a/config_amwg_default_plots.yaml b/config_amwg_default_plots.yaml deleted file mode 100644 index 7323d3b31..000000000 --- a/config_amwg_default_plots.yaml +++ /dev/null @@ -1,470 +0,0 @@ -#============================== -# config_amwg_default_plots.yaml - -# This config file contains the standard set of variables and plots used for -# evaluating CAM simulations in the AMWG working group. - -#Currently, if one is on NCAR's Casper or -#Cheyenne machine, then only the diagnostic output -#paths are needed, at least to perform a quick test -#run (these are indicated with "MUST EDIT" comments). -#Running these diagnostics on a different machine, -#or with a different, non-example simulation, will -#require additional modifications. -# -#Config file Keywords: -#-------------------- -# -#1. Using ${xxx} will substitute that text with the -# variable referenced by xxx. For example: -# -# cam_case_name: cool_run -# cam_climo_loc: /some/where/${cam_case_name} -# -# will set "cam_climo_loc" in the diagnostics package to: -# /some/where/cool_run -# -# Please note that currently this will only work if the -# variable only exists in one location in the file. -# -#2. Using ${.xxx} will do the same as -# keyword 1 above, but specifies which sub-section the -# variable is coming from, which is necessary for variables -# that are repeated in different subsections. For example: -# -# diag_basic_info: -# cam_climo_loc: /some/where/${diag_cam_climo.start_year} -# -# diag_cam_climo: -# start_year: 1850 -# -# will set "cam_climo_loc" in the diagnostics package to: -# /some/where/1850 -# -#Finally, please note that for both 1 and 2 the keywords must be lowercase. -#This is because future developments will hopefully use other keywords -#that are uppercase. Also please avoid using periods (".") in variable -#names, as this will likely cause issues with the current file parsing -#system. -#-------------------- -# -##============================== -# -# This file doesn't (yet) read environment variables, so the user must -# set this themselves. It is also a good idea to search the doc for 'user' -# to see what default paths are being set for output/working files. -# -# Note that the string 'USER-NAME-NOT-SET' is used in the jupyter script -# to check for a failure to customize -# -user: 'USER-NAME-NOT-SET' - - -#This first set of variables specify basic info used by all diagnostic runs: -diag_basic_info: - - #Is this a model vs observations comparison? - #If "false" or missing, then a model-model comparison is assumed: - compare_obs: false - - #Generate HTML website (assumed false if missing): - #Note: The website files themselves will be located in the path - #specified by "cam_diag_plot_loc", under the "/website" subdirectory, - #where "" is the subdirectory created for this particular diagnostics run - #(usually "case_vs_obs_XXX" or "case_vs_baseline_XXX"). - create_html: true - - #Location of observational datasets: - #Note: this only matters if "compare_obs" is true and the path - #isn't specified in the variable defaults file. - obs_data_loc: /glade/campaign/cgd/amp/amwg/ADF_obs - - #Location where re-gridded and interpolated CAM climatology files are stored: - cam_regrid_loc: /glade/derecho/scratch/${user}/ADF/regrid - - #Overwrite CAM re-gridded files? - #If false, or missing, then regridding will be skipped for regridded variables - #that already exist in "cam_regrid_loc": - cam_overwrite_regrid: false - - #Location where diagnostic plots are stored: - cam_diag_plot_loc: /glade/derecho/scratch/${user}/ADF/plots - - #Location of ADF variable plotting defaults YAML file: - #If left blank or missing, ADF/lib/adf_variable_defaults.yaml will be used - #Uncomment and change path for custom variable defaults file - #defaults_file: /some/path/to/defaults/file.yaml - - #Vertical pressure levels (in hPa) on which to plot 3-D variables - #when using horizontal (e.g. lat/lon) map projections. - #If this config option is missing, then no 3-D variables will be plotted on - #horizontal maps. Please note too that pressure levels must currently match - #what is available in the observations file in order to be plotted in a - #model vs obs run: - plot_press_levels: [200,850] - - #Longitude line on which to center all lat/lon maps. - #If this config option is missing then the central - #longitude will default to 180 degrees E. - central_longitude: 180 - - #Number of processors on which to run the ADF. - #If this config variable isn't present then - #the ADF defaults to one processor. Also, if - #you set it to "*" then it will default - #to all of the processors available on a - #single node/machine: - num_procs: 8 - - #If set to true, then redo all plots even if they already exist. - #If set to false, then if a plot is found it will be skipped: - redo_plot: false - - -#This second set of variables provides info for the CAM simulation(s) being diagnosed: -diag_cam_climo: - - # History file list of strings to match - # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] - # Only affects timeseries as everything else uses the created timeseries - # Default: - hist_str: cam.h0a - - #Calculate climatologies? - #If false, the climatology files will not be created: - calc_cam_climo: true - - #Overwrite CAM climatology files? - #If false, or not prsent, then already existing climatology files will be skipped: - cam_overwrite_climo: false - - #Name of CAM case (or CAM run name): - cam_case_name: b.e23_alpha17f.BLT1850.ne30_t232.098 - - #Case nickname - #NOTE: if nickname starts with '0' - nickname must be in quotes! - # ie '026a' as opposed to 026a - #If missing or left blank, will default to cam_case_name - case_nickname: #cool nickname - - #Location of CAM history (h0) files: - #Example test files - cam_hist_loc: /glade/campaign/cgd/amp/amwg/ADF_test_cases/${diag_cam_climo.cam_case_name} - - #Location of CAM climatologies (to be created and then used by this script) - cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/climo - - #model year when time series files should start: - #Note: Leaving this entry blank will make time series - # start at earliest available year. - start_year: 10 - - #model year when time series files should end: - #Note: Leaving this entry blank will make time series - # end at latest available year. - end_year: 14 - - #Do time series files exist? - #If True, then diagnostics assumes that model files are already time series. - #If False, or if simply not present, then diagnostics will attempt to create - #time series files from history (time-slice) files: - cam_ts_done: false - - #Save interim time series files? - #WARNING: This can take up a significant amount of space, - # but will save processing time the next time - cam_ts_save: true - - #Overwrite time series files, if found? - #If set to false, then time series creation will be skipped if files are found: - cam_overwrite_ts: false - - #Location where time series files are (or will be) stored: - cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/ts - - #TEM diagnostics - #--------------- - #TEM history file number - #If missing or blank, ADF will default to h4 - tem_hist_str: cam.h4 - - #Location where TEM files are stored: - #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! - cam_tem_loc: /glade/derecho/scratch/${user}/${diag_cam_climo.cam_case_name}/tem/ - - #Overwrite TEM files, if found? - #If set to false, then TEM creation will be skipped if files are found: - overwrite_tem: false - - #---------------------- - -#This third set of variables provide info for the CAM baseline climatologies. -#This only matters if "compare_obs" is false: -diag_cam_baseline_climo: - - # History file list of strings to match - # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] - # Only affects timeseries as everything else uses the created timeseries - # Default: - hist_str: cam.h0a - - #Calculate cam baseline climatologies? - #If false, the climatology files will not be created: - calc_cam_climo: true - - #Overwrite CAM climatology files? - #If false, or not present, then already existing climatology files will be skipped: - cam_overwrite_climo: false - - #Name of CAM baseline case: - cam_case_name: b.e23_alpha17f.BLT1850.ne30_t232.093 - - #Baseline case nickname - #NOTE: if nickname starts with '0' - nickname must be in quotes! - # ie '026a' as opposed to 026a - #If missing or left blank, will default to cam_case_name - case_nickname: #cool nickname - - #Location of CAM baseline history (h0) files: - #Example test files - cam_hist_loc: /glade/campaign/cgd/amp/amwg/ADF_test_cases/${diag_cam_baseline_climo.cam_case_name} - - #Location of baseline CAM climatologies: - cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/climo - - #model year when time series files should start: - #Note: Leaving this entry blank will make time series - # start at earliest available year. - start_year: 10 - - #model year when time series files should end: - #Note: Leaving this entry blank will make time series - # end at latest available year. - end_year: 14 - - #Do time series files need to be generated? - #If True, then diagnostics assumes that model files are already time series. - #If False, or if simply not present, then diagnostics will attempt to create - #time series files from history (time-slice) files: - cam_ts_done: false - - #Save interim time series files for baseline run? - #WARNING: This can take up a significant amount of space: - cam_ts_save: true - - #Overwrite baseline time series files, if found? - #If set to false, then time series creation will be skipped if files are found: - cam_overwrite_ts: false - - #Location where time series files are (or will be) stored: - cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/ts - - #TEM diagnostics - #--------------- - #TEM history file number - #If missing or blank, ADF will default to h4 - tem_hist_str: cam.h4 - - #Location where TEM files are stored: - #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! - cam_tem_loc: /glade/derecho/scratch/${user}/${diag_cam_baseline_climo.cam_case_name}/tem/ - - #Overwrite TEM files, if found? - #If set to false, then TEM creation will be skipped if files are found: - overwrite_tem: false - -#This fourth set of variables provides settings for calling the Climate Variability -# Diagnostics Package (CVDP). If cvdp_run is set to true the CVDP will be set up and -# run in background mode, likely completing after the ADF has completed. -# If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed -# in the diag_var_list variable listing. -# For more CVDP information: https://www.cesm.ucar.edu/working_groups/CVC/cvdp/ -diag_cvdp_info: - - # Run the CVDP on the listed run(s)? - cvdp_run: false - - # CVDP code path, sets the location of the CVDP codebase - # CGD systems path = /home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - # CISL systems path = /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - # github location = https://github.com/NCAR/CVDP-ncl - cvdp_codebase_loc: /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - - # Location where cvdp codebase will be copied to and diagnostic plots will be stored - cvdp_loc: /glade/derecho/scratch/${user}/ADF/cvdp/ - - # tar up CVDP results? - cvdp_tar: false - -# This set of variables provides settings for calling NOAA's -# Model Diagnostic Task Force (MDTF) diagnostic package. -# https://github.com/NOAA-GFDL/MDTF-diagnostics -# -# If mdtf_run: true, the MDTF will be set up and -# run in background mode, likely completing after the ADF has completed. -# -# WARNING: This currently only runs on CASPER (not derecho) -# -# The variables required depend on the diagnostics (PODs) selected. -# AMWG-developed PODS and their required variables: -# (Note that PRECT can be computed from PRECC & PRECL) -# - MJO_suite: daily PRECT, FLUT, U850, U200, V200 (all required) -# - Wheeler-Kiladis Wavenumber Frequency Spectra: daily PRECT, FLUT, U200, U850, OMEGA500 -# (will use what is available) -# - Blocking (Rich Neale): daily OMEGA500 -# - Precip Diurnal Cycle (Rich Neale): 3-hrly PRECT -# -# Many other diagnostics are available; see -# https://mdtf-diagnostics.readthedocs.io/en/main/sphinx/start_overview.html - -# -diag_mdtf_info: - # Run the MDTF on the model cases - mdtf_run: false - - # The file that will be written by ADF to input to MDTF. Call this whatever you want. - mdtf_input_settings_filename : mdtf_input.json - - ## MDTF code path, sets the location of the MDTF codebase and pre-compiled conda envs - # CHANGE if you have any: your own MDTF code, installed conda envs and/or obs_data - - mdtf_codebase_path : /glade/campaign/cgd/amp/amwg/mdtf - mdtf_codebase_loc : ${mdtf_codebase_path}/MDTF-diagnostics.v3.1.20230817.ADF - conda_root : /glade/u/apps/opt/conda - conda_env_root : ${mdtf_codebase_path}/miniconda2/envs.MDTFv3.1.20230412/ - OBS_DATA_ROOT : ${mdtf_codebase_path}/obs_data - - # SET this to a writable dir. The ADF will place ts files here for the MDTF to read (adds the casename) - MODEL_DATA_ROOT : ${diag_cam_climo.cam_ts_loc}/mdtf/inputdata/model - - # Choose diagnostics (PODs). Full list of available PODs: https://github.com/NOAA-GFDL/MDTF-diagnostics - pod_list : [ "MJO_suite" ] - - # Intermediate/output file settings - make_variab_tar: false # tar up MDTF results - save_ps : false # save postscript figures in addition to bitmaps - save_nc : false # save netCDF files of processed data (recommend true when starting with new model data) - overwrite: true # overwrite results in OUTPUT_DIR; otherwise results will be saved under a unique name - - # Settings used in debugging: - verbose : 3 # Log verbosity level. - test_mode: false # Set to true for framework test. Data is fetched but PODs are not run. - dry_run : false # Framework test. No external commands are run and no remote data is copied. Implies test_mode. - - # Settings that shouldn't change in ADF implementation for now - data_type : single_run # single_run or multi_run (only works with single right now) - data_manager : Local_File # Fetch data or it is local? - environment_manager : Conda # Manage dependencies - - - -#+++++++++++++++++++++++++++++++++++++++++++++++++++ -#These variables below only matter if you are using -#a non-standard method, or are adding your own -#diagnostic scripts. -#+++++++++++++++++++++++++++++++++++++++++++++++++++ - -#Note: If you want to pass arguments to a particular script, you can -#do it like so (using the "averaging_example" script in this case): -# - {create_climo_files: {kwargs: {clobber: true}}} - -#Name of time-averaging scripts being used to generate climatologies. -#These scripts must be located in "scripts/averaging": -time_averaging_scripts: - - create_climo_files - #- create_TEM_files #To generate TEM files, please un-comment - -#Name of regridding scripts being used. -#These scripts must be located in "scripts/regridding": -regridding_scripts: - - regrid_and_vert_interp - -#List of analysis scripts being used. -#These scripts must be located in "scripts/analysis": -analysis_scripts: - - amwg_table - #- aerosol_gas_tables - -#List of plotting scripts being used. -#These scripts must be located in "scripts/plotting": -plotting_scripts: - - global_latlon_map - - global_latlon_vect_map - - zonal_mean - - polar_map - - cam_taylor_diagram - - ozone_diagnostics - #- tape_recorder - #- tem - #- regional_map_multicase #To use this please un-comment and fill-out - #the "region_multicase" section below - -#List of CAM variables that will be processesd: -#If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed -diag_var_list: - - AODDUST - - AODVIS - - CLDHGH - - CLDICE - - CLDLIQ - - CLDLOW - - CLDMED - - CLDTOT - - CLOUD - - FLNS - - FLNT - - FLNTC - - FSNS - - FSNT - - FSNTC - - LHFLX - - LWCF - - OMEGA500 - - PBLH - - PRECT - - PS - - PSL - - QFLX - - RELHUM - - SHFLX - - SST - - SWCF - - T - - TAUX - - TAUY - - TGCLDIWP - - TGCLDLWP - - TMQ - - TREFHT - - TS - - U - - U10 - - ICEFRAC - - OCNFRAC - - LANDFRAC - - O3 - -# -# MDTF recommended variables -# - OMEGA -# - PRECT -# - PS -# - PSL -# - U200 -# - U850 -# - V200 -# - V850 - -# Options for multi-case regional contour plots (./plotting/regional_map_multicase.py) -# region_multicase: -# region_spec: [slat, nlat, wlon, elon] -# region_time_option: # If calendar, will look for specified years. If zeroanchor will use a nyears starting from year_offset from the beginning of timeseries -# region_start_year: -# region_end_year: -# region_nyear: -# region_year_offset: -# region_month: -# region_season: -# region_variables: - -#END OF FILE \ No newline at end of file diff --git a/config_cam_baseline_example.yaml b/config_cam_baseline_example.yaml deleted file mode 100644 index d9f3ad4c2..000000000 --- a/config_cam_baseline_example.yaml +++ /dev/null @@ -1,534 +0,0 @@ -#============================== -#config_cam_baseline_example.yaml - -#This is the main CAM diagnostics config file -#for doing comparisons of a CAM run against -#another CAM run, or a CAM baseline simulation. - -#Currently, if one is on NCAR's Casper or -#Cheyenne machine, then only the diagnostic output -#paths are needed, at least to perform a quick test -#run (these are indicated with "MUST EDIT" comments). -#Running these diagnostics on a different machine, -#or with a different, non-example simulation, will -#require additional modifications. -# -#Config file Keywords: -#-------------------- -# -#1. Using ${xxx} will substitute that text with the -# variable referenced by xxx. For example: -# -# cam_case_name: cool_run -# cam_climo_loc: /some/where/${cam_case_name} -# -# will set "cam_climo_loc" in the diagnostics package to: -# /some/where/cool_run -# -# Please note that currently this will only work if the -# variable only exists in one location in the file. -# -#2. Using ${.xxx} will do the same as -# keyword 1 above, but specifies which sub-section the -# variable is coming from, which is necessary for variables -# that are repeated in different subsections. For example: -# -# diag_basic_info: -# cam_climo_loc: /some/where/${diag_cam_climo.start_year} -# -# diag_cam_climo: -# start_year: 1850 -# -# will set "cam_climo_loc" in the diagnostics package to: -# /some/where/1850 -# -#Finally, please note that for both 1 and 2 the keywords must be lowercase. -#This is because future developments will hopefully use other keywords -#that are uppercase. Also please avoid using periods (".") in variable -#names, as this will likely cause issues with the current file parsing -#system. -#-------------------- -# -##============================== -# -# This file doesn't (yet) read environment variables, so the user must -# set this themselves. It is also a good idea to search the doc for 'user' -# to see what default paths are being set for output/working files. -# -# Note that the string 'USER-NAME-NOT-SET' is used in the jupyter script -# to check for a failure to customize -# -user: 'USER-NAME-NOT-SET' - - -#This first set of variables specify basic info used by all diagnostic runs: -diag_basic_info: - - #Is this a model vs observations comparison? - #If "false" or missing, then a model-model comparison is assumed: - compare_obs: false - - #Generate HTML website (assumed false if missing): - #Note: The website files themselves will be located in the path - #specified by "cam_diag_plot_loc", under the "/website" subdirectory, - #where "" is the subdirectory created for this particular diagnostics run - #(usually "case_vs_obs_XXX" or "case_vs_baseline_XXX"). - create_html: true - - #Location of observational datasets: - #Note: this only matters if "compare_obs" is true and the path - #isn't specified in the variable defaults file. - obs_data_loc: /glade/campaign/cgd/amp/amwg/ADF_obs - - #Location where re-gridded and interpolated CAM climatology files are stored: - cam_regrid_loc: /glade/derecho/scratch/${user}/ADF/regrid - - #Overwrite CAM re-gridded files? - #If false, or missing, then regridding will be skipped for regridded variables - #that already exist in "cam_regrid_loc": - cam_overwrite_regrid: false - - #Location where diagnostic plots are stored: - cam_diag_plot_loc: /glade/derecho/scratch/${user}/ADF/plots - - #Location of ADF variable plotting defaults YAML file: - #If left blank or missing, ADF/lib/adf_variable_defaults.yaml will be used - #Uncomment and change path for custom variable defaults file - #defaults_file: /some/path/to/defaults/file.yaml - - #Vertical pressure levels (in hPa) on which to plot 3-D variables - #when using horizontal (e.g. lat/lon) map projections. - #If this config option is missing, then no 3-D variables will be plotted on - #horizontal maps. Please note too that pressure levels must currently match - #what is available in the observations file in order to be plotted in a - #model vs obs run: - plot_press_levels: [200,850] - - #Longitude line on which to center all lat/lon maps. - #If this config option is missing then the central - #longitude will default to 180 degrees E. - central_longitude: 180 - - #Number of processors on which to run the ADF. - #If this config variable isn't present then - #the ADF defaults to one processor. Also, if - #you set it to "*" then it will default - #to all of the processors available on a - #single node/machine: - num_procs: 8 - - #If set to true, then redo all plots even if they already exist. - #If set to false, then if a plot is found it will be skipped: - redo_plot: false - -#This second set of variables provides info for the CAM simulation(s) being diagnosed: -diag_cam_climo: - - # History file list of strings to match - # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] - # Only affects timeseries as everything else uses the created timeseries - # Default: - hist_str: cam.h0a - - #Calculate climatologies? - #If false, the climatology files will not be created: - calc_cam_climo: true - - #Overwrite CAM climatology files? - #If false, or not prsent, then already existing climatology files will be skipped: - cam_overwrite_climo: false - - #Name of CAM case (or CAM run name): - cam_case_name: b.e23_alpha17f.BLT1850.ne30_t232.098 - - #Case nickname - #NOTE: if nickname starts with '0' - nickname must be in quotes! - # ie '026a' as opposed to 026a - #If missing or left blank, will default to cam_case_name - case_nickname: #cool nickname - - #Location of CAM history (h0) files: - #Example test files - cam_hist_loc: /glade/campaign/cgd/amp/amwg/ADF_test_cases/${diag_cam_climo.cam_case_name} - - #Location of CAM climatologies (to be created and then used by this script) - cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/climo - - #model year when time series files should start: - #Note: Leaving this entry blank will make time series - # start at earliest available year. - start_year: 10 - - #model year when time series files should end: - #Note: Leaving this entry blank will make time series - # end at latest available year. - end_year: 14 - - #Do time series files exist? - #If True, then diagnostics assumes that model files are already time series. - #If False, or if simply not present, then diagnostics will attempt to create - #time series files from history (time-slice) files: - cam_ts_done: false - - #Save interim time series files? - #WARNING: This can take up a significant amount of space, - # but will save processing time the next time - cam_ts_save: true - - #Overwrite time series files, if found? - #If set to false, then time series creation will be skipped if files are found: - cam_overwrite_ts: false - - #Location where time series files are (or will be) stored: - cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/ts - - #TEM diagnostics - #--------------- - #TEM history file number - #If missing or blank, ADF will default to h4 - tem_hist_str: cam.h4 - - #Location where TEM files are stored: - #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! - cam_tem_loc: /glade/derecho/scratch/${user}/${diag_cam_climo.cam_case_name}/tem/ - - #Overwrite TEM files, if found? - #If set to false, then TEM creation will be skipped if files are found: - overwrite_tem: false - - #---------------------- - - #You can alternatively provide a list of cases, which will make the ADF - #apply the same diagnostics to each case separately in a single ADF session. - #All of the config variables below show how it is done, and are the only ones - #that need to be lists. This also automatically enables the generation of - #a "main_website" in "cam_diag_plot_loc" that brings all of the different cases - #together under a single website. - - #Also please note that config keywords cannot currently be used in list mode. - - #cam_case_name: - # - b.e23_alpha17f.BLT1850.ne30_t232.098 - # - b.e23_alpha17f.BLT1850.ne30_t232.095 - - #Case nickname - #NOTE: if nickname starts with '0' - nickname must be in quotes! - # ie '026a' as opposed to 026a - #If missing or left blank, will default to cam_case_name - #case_nickname: - # - cool nickname - # - cool nickname 2 - - #calc_cam_climo: - # - true - # - true - - #cam_overwrite_climo: - # - false - # - false - - #cam_hist_loc: - # - /glade/campaign/cgd/amp/amwg/ADF_test_cases/b.e23_alpha17f.BLT1850.ne30_t232.098 - # - /glade/campaign/cgd/amp/amwg/ADF_test_cases/b.e23_alpha17f.BLT1850.ne30_t232.095 - - #cam_climo_loc: - # - /some/where/you/want/to/have/climo_files/ #MUST EDIT! - # - /the/same/or/some/other/climo/files/location - - #start_year: - # - 10 - # - 10 - - #end_year: - # - 14 - # - 14 - - #cam_ts_done: - # - false - # - false - - #cam_ts_save: - # - true - # - true - - #cam_overwrite_ts: - # - false - # - false - - #cam_ts_loc: - # - /some/where/you/want/to/have/time_series_files - # - /same/or/different/place/you/want/files - - #TEM diagnostics - #--------------- - #TEM history file number - #If missing or blank, ADF will default to h4 - #tem_hist_str: - # - cam.h4 - # - cam.h# - - #Location where TEM files are stored: - #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! - #cam_tem_loc: - # - /some/where/you/want/to/have/TEM_files/ - # - /same/or/different/place/you/want/TEM_files/ - - #Overwrite TEM files, if found? - #If set to false, then TEM creation will be skipped if files are found: - #overwrite_tem: - # - false - # - true - - #---------------------- - - -#This third set of variables provide info for the CAM baseline climatologies. -#This only matters if "compare_obs" is false: -diag_cam_baseline_climo: - - # History file list of strings to match - # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] - # Only affects timeseries as everything else uses the created timeseries - # Default: - hist_str: cam.h0a - - #Calculate cam baseline climatologies? - #If false, the climatology files will not be created: - calc_cam_climo: true - - #Overwrite CAM climatology files? - #If false, or not present, then already existing climatology files will be skipped: - cam_overwrite_climo: false - - #Name of CAM baseline case: - cam_case_name: b.e23_alpha17f.BLT1850.ne30_t232.093 - - #Baseline case nickname - #NOTE: if nickname starts with '0' - nickname must be in quotes! - # ie '026a' as opposed to 026a - #If missing or left blank, will default to cam_case_name - case_nickname: #cool nickname - - #Location of CAM baseline history (h0) files: - #Example test files - cam_hist_loc: /glade/campaign/cgd/amp/amwg/ADF_test_cases/${diag_cam_baseline_climo.cam_case_name} - - #Location of baseline CAM climatologies: - cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/climo - - #model year when time series files should start: - #Note: Leaving this entry blank will make time series - # start at earliest available year. - start_year: 10 - - #model year when time series files should end: - #Note: Leaving this entry blank will make time series - # end at latest available year. - end_year: 14 - - #Do time series files need to be generated? - #If True, then diagnostics assumes that model files are already time series. - #If False, or if simply not present, then diagnostics will attempt to create - #time series files from history (time-slice) files: - cam_ts_done: false - - #Save interim time series files for baseline run? - #WARNING: This can take up a significant amount of space: - cam_ts_save: true - - #Overwrite baseline time series files, if found? - #If set to false, then time series creation will be skipped if files are found: - cam_overwrite_ts: false - - #Location where time series files are (or will be) stored: - cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/ts - - #TEM diagnostics - #--------------- - #TEM history file number - #If missing or blank, ADF will default to h4 - tem_hist_str: cam.h4 - - #Location where TEM files are stored: - #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! - cam_tem_loc: /glade/derecho/scratch/${user}/${diag_cam_baseline_climo.cam_case_name}/tem/ - - #Overwrite TEM files, if found? - #If set to false, then TEM creation will be skipped if files are found: - overwrite_tem: false - - -#This fourth set of variables provides settings for calling the Climate Variability -# Diagnostics Package (CVDP). If cvdp_run is set to true the CVDP will be set up and -# run in background mode, likely completing after the ADF has completed. -# If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed -# in the diag_var_list variable listing. -# For more CVDP information: https://www.cesm.ucar.edu/working_groups/CVC/cvdp/ -diag_cvdp_info: - - # Run the CVDP on the listed run(s)? - cvdp_run: false - - # CVDP code path, sets the location of the CVDP codebase - # CGD systems path = /home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - # CISL systems path = /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - # github location = https://github.com/NCAR/CVDP-ncl - cvdp_codebase_loc: /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - - # Location where cvdp codebase will be copied to and diagnostic plots will be stored - cvdp_loc: /glade/derecho/scratch/${user}/ADF/cvdp/ - - # tar up CVDP results? - cvdp_tar: false - -# This set of variables provides settings for calling NOAA's -# Model Diagnostic Task Force (MDTF) diagnostic package. -# https://github.com/NOAA-GFDL/MDTF-diagnostics -# -# If mdtf_run: true, the MDTF will be set up and -# run in background mode, likely completing after the ADF has completed. -# -# WARNING: This currently only runs on CASPER (not derecho) -# -# The variables required depend on the diagnostics (PODs) selected. -# AMWG-developed PODS and their required variables: -# (Note that PRECT can be computed from PRECC & PRECL) -# - MJO_suite: daily PRECT, FLUT, U850, U200, V200 (all required) -# - Wheeler-Kiladis Wavenumber Frequency Spectra: daily PRECT, FLUT, U200, U850, OMEGA500 -# (will use what is available) -# - Blocking (Rich Neale): daily OMEGA500 -# - Precip Diurnal Cycle (Rich Neale): 3-hrly PRECT -# -# Many other diagnostics are available; see -# https://mdtf-diagnostics.readthedocs.io/en/main/sphinx/start_overview.html - -# -diag_mdtf_info: - # Run the MDTF on the model cases - mdtf_run: false - - # The file that will be written by ADF to input to MDTF. Call this whatever you want. - mdtf_input_settings_filename : mdtf_input.json - - ## MDTF code path, sets the location of the MDTF codebase and pre-compiled conda envs - # CHANGE if you have any: your own MDTF code, installed conda envs and/or obs_data - - mdtf_codebase_path : /glade/campaign/cgd/amp/amwg/mdtf - mdtf_codebase_loc : ${mdtf_codebase_path}/MDTF-diagnostics.v3.1.20230817.ADF - conda_root : /glade/u/apps/opt/conda - conda_env_root : ${mdtf_codebase_path}/miniconda2/envs.MDTFv3.1.20230412/ - OBS_DATA_ROOT : ${mdtf_codebase_path}/obs_data - - # SET this to a writable dir. The ADF will place ts files here for the MDTF to read (adds the casename) - MODEL_DATA_ROOT : ${diag_cam_climo.cam_ts_loc}/mdtf/inputdata/model - - # Choose diagnostics (PODs). Full list of available PODs: https://github.com/NOAA-GFDL/MDTF-diagnostics - pod_list : [ "MJO_suite" ] - - # Intermediate/output file settings - make_variab_tar: false # tar up MDTF results - save_ps : false # save postscript figures in addition to bitmaps - save_nc : false # save netCDF files of processed data (recommend true when starting with new model data) - overwrite: true # overwrite results in OUTPUT_DIR; otherwise results will be saved under a unique name - - # Settings used in debugging: - verbose : 3 # Log verbosity level. - test_mode: false # Set to true for framework test. Data is fetched but PODs are not run. - dry_run : false # Framework test. No external commands are run and no remote data is copied. Implies test_mode. - - # Settings that shouldn't change in ADF implementation for now - data_type : single_run # single_run or multi_run (only works with single right now) - data_manager : Local_File # Fetch data or it is local? - environment_manager : Conda # Manage dependencies - - - -#+++++++++++++++++++++++++++++++++++++++++++++++++++ -#These variables below only matter if you are using -#a non-standard method, or are adding your own -#diagnostic scripts. -#+++++++++++++++++++++++++++++++++++++++++++++++++++ - -#Note: If you want to pass arguments to a particular script, you can -#do it like so (using the "averaging_example" script in this case): -# - {create_climo_files: {kwargs: {clobber: true}}} - -#Name of time-averaging scripts being used to generate climatologies. -#These scripts must be located in "scripts/averaging": -time_averaging_scripts: - - create_climo_files - #- create_TEM_files #To generate TEM files, please un-comment - -#Name of regridding scripts being used. -#These scripts must be located in "scripts/regridding": -regridding_scripts: - - regrid_and_vert_interp - -#List of analysis scripts being used. -#These scripts must be located in "scripts/analysis": -analysis_scripts: - - amwg_table - #- aerosol_gas_tables - -#List of plotting scripts being used. -#These scripts must be located in "scripts/plotting": -plotting_scripts: - - global_latlon_map - - global_latlon_vect_map - - zonal_mean - - meridional_mean - - polar_map - - cam_taylor_diagram - - qbo - - ozone_diagnostics - #- tape_recorder - #- tem - #- regional_map_multicase #To use this please un-comment and fill-out - #the "region_multicase" section below - -#List of CAM variables that will be processesd: -#If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed -diag_var_list: - - SWCF - - LWCF - - PRECC - - PRECL - - PSL - - Q - - U - - T - - RELHUM - - TREFHT - - TS - - TAUX - - TAUY - - FSNT - - FLNT - - LANDFRAC - - O3 - -# -# MDTF recommended variables -# - FLUT -# - OMEGA500 -# - PRECT -# - PS -# - PSL -# - U200 -# - U850 -# - V200 -# - V850 - -# Options for multi-case regional contour plots (./plotting/regional_map_multicase.py) -# region_multicase: -# region_spec: [slat, nlat, wlon, elon] -# region_time_option: # If calendar, will look for specified years. If zeroanchor will use a nyears starting from year_offset from the beginning of timeseries -# region_start_year: -# region_end_year: -# region_nyear: -# region_year_offset: -# region_month: -# region_season: -# region_variables: - -#END OF FILE diff --git a/config_cam_baseline_example_testENSO.yaml b/config_cam_baseline_example_testENSO.yaml deleted file mode 100644 index 8113000ec..000000000 --- a/config_cam_baseline_example_testENSO.yaml +++ /dev/null @@ -1,537 +0,0 @@ -#============================== -#config_cam_baseline_example.yaml - -#This is the main CAM diagnostics config file -#for doing comparisons of a CAM run against -#another CAM run, or a CAM baseline simulation. - -#Currently, if one is on NCAR's Casper or -#Cheyenne machine, then only the diagnostic output -#paths are needed, at least to perform a quick test -#run (these are indicated with "MUST EDIT" comments). -#Running these diagnostics on a different machine, -#or with a different, non-example simulation, will -#require additional modifications. -# -#Config file Keywords: -#-------------------- -# -#1. Using ${xxx} will substitute that text with the -# variable referenced by xxx. For example: -# -# cam_case_name: cool_run -# cam_climo_loc: /some/where/${cam_case_name} -# -# will set "cam_climo_loc" in the diagnostics package to: -# /some/where/cool_run -# -# Please note that currently this will only work if the -# variable only exists in one location in the file. -# -#2. Using ${.xxx} will do the same as -# keyword 1 above, but specifies which sub-section the -# variable is coming from, which is necessary for variables -# that are repeated in different subsections. For example: -# -# diag_basic_info: -# cam_climo_loc: /some/where/${diag_cam_climo.start_year} -# -# diag_cam_climo: -# start_year: 1850 -# -# will set "cam_climo_loc" in the diagnostics package to: -# /some/where/1850 -# -#Finally, please note that for both 1 and 2 the keywords must be lowercase. -#This is because future developments will hopefully use other keywords -#that are uppercase. Also please avoid using periods (".") in variable -#names, as this will likely cause issues with the current file parsing -#system. -#-------------------- -# -##============================== -# -# This file doesn't (yet) read environment variables, so the user must -# set this themselves. It is also a good idea to search the doc for 'user' -# to see what default paths are being set for output/working files. -# -# Note that the string 'USER-NAME-NOT-SET' is used in the jupyter script -# to check for a failure to customize -# -user: 'mdfowler' - - -#This first set of variables specify basic info used by all diagnostic runs: -diag_basic_info: - - #Is this a model vs observations comparison? - #If "false" or missing, then a model-model comparison is assumed: - compare_obs: false - - #Generate HTML website (assumed false if missing): - #Note: The website files themselves will be located in the path - #specified by "cam_diag_plot_loc", under the "/website" subdirectory, - #where "" is the subdirectory created for this particular diagnostics run - #(usually "case_vs_obs_XXX" or "case_vs_baseline_XXX"). - create_html: true - - #Location of observational datasets: - #Note: this only matters if "compare_obs" is true and the path - #isn't specified in the variable defaults file. - obs_data_loc: /glade/campaign/cgd/amp/amwg/ADF_obs - - #Location where re-gridded and interpolated CAM climatology files are stored: - cam_regrid_loc: /glade/derecho/scratch/${user}/ADF/regrid - - #Overwrite CAM re-gridded files? - #If false, or missing, then regridding will be skipped for regridded variables - #that already exist in "cam_regrid_loc": - cam_overwrite_regrid: false - - #Location where diagnostic plots are stored: - cam_diag_plot_loc: /glade/derecho/scratch/${user}/ADF/plots - - #Location of ADF variable plotting defaults YAML file: - #If left blank or missing, ADF/lib/adf_variable_defaults.yaml will be used - #Uncomment and change path for custom variable defaults file - #defaults_file: /some/path/to/defaults/file.yaml - - #Vertical pressure levels (in hPa) on which to plot 3-D variables - #when using horizontal (e.g. lat/lon) map projections. - #If this config option is missing, then no 3-D variables will be plotted on - #horizontal maps. Please note too that pressure levels must currently match - #what is available in the observations file in order to be plotted in a - #model vs obs run: - plot_press_levels: [200,850] - - #Longitude line on which to center all lat/lon maps. - #If this config option is missing then the central - #longitude will default to 180 degrees E. - central_longitude: 180 - - #Number of processors on which to run the ADF. - #If this config variable isn't present then - #the ADF defaults to one processor. Also, if - #you set it to "*" then it will default - #to all of the processors available on a - #single node/machine: - num_procs: 8 - - #If set to true, then redo all plots even if they already exist. - #If set to false, then if a plot is found it will be skipped: - redo_plot: true - -#This second set of variables provides info for the CAM simulation(s) being diagnosed: -diag_cam_climo: - - # History file list of strings to match - # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] - # Only affects timeseries as everything else uses the created timeseries - # Default: - hist_str: cam.h0a - - #Calculate climatologies? - #If false, the climatology files will not be created: - calc_cam_climo: true - - #Overwrite CAM climatology files? - #If false, or not prsent, then already existing climatology files will be skipped: - cam_overwrite_climo: false - - #Name of CAM case (or CAM run name): - cam_case_name: b.e30_alpha06b.B1850C_LTso.ne30_t232_wgx3.132 - - #Case nickname - #NOTE: if nickname starts with '0' - nickname must be in quotes! - # ie '026a' as opposed to 026a - #If missing or left blank, will default to cam_case_name - case_nickname: '132' - - #Location of CAM history (h0) files: - #Example test files - # cam_hist_loc: /glade/campaign/cgd/amp/amwg/ADF_test_cases/${diag_cam_climo.cam_case_name} - cam_hist_loc: /glade/derecho/scratch/hannay/archive//b.e30_alpha06b.B1850C_LTso.ne30_t232_wgx3.132/atm/hist - - #Location of CAM climatologies (to be created and then used by this script) - cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/climo - - #model year when time series files should start: - #Note: Leaving this entry blank will make time series - # start at earliest available year. - start_year: 2 - - #model year when time series files should end: - #Note: Leaving this entry blank will make time series - # end at latest available year. - end_year: 44 - - #Do time series files exist? - #If True, then diagnostics assumes that model files are already time series. - #If False, or if simply not present, then diagnostics will attempt to create - #time series files from history (time-slice) files: - cam_ts_done: false - - #Save interim time series files? - #WARNING: This can take up a significant amount of space, - # but will save processing time the next time - cam_ts_save: true - - #Overwrite time series files, if found? - #If set to false, then time series creation will be skipped if files are found: - cam_overwrite_ts: false - - #Location where time series files are (or will be) stored: - cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/ts - - #TEM diagnostics - #--------------- - #TEM history file number - #If missing or blank, ADF will default to h4 - tem_hist_str: cam.h4 - - #Location where TEM files are stored: - #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! - cam_tem_loc: /glade/derecho/scratch/${user}/${diag_cam_climo.cam_case_name}/tem/ - - #Overwrite TEM files, if found? - #If set to false, then TEM creation will be skipped if files are found: - overwrite_tem: false - - #---------------------- - - #You can alternatively provide a list of cases, which will make the ADF - #apply the same diagnostics to each case separately in a single ADF session. - #All of the config variables below show how it is done, and are the only ones - #that need to be lists. This also automatically enables the generation of - #a "main_website" in "cam_diag_plot_loc" that brings all of the different cases - #together under a single website. - - #Also please note that config keywords cannot currently be used in list mode. - - #cam_case_name: - # - b.e23_alpha17f.BLT1850.ne30_t232.098 - # - b.e23_alpha17f.BLT1850.ne30_t232.095 - - #Case nickname - #NOTE: if nickname starts with '0' - nickname must be in quotes! - # ie '026a' as opposed to 026a - #If missing or left blank, will default to cam_case_name - #case_nickname: - # - cool nickname - # - cool nickname 2 - - #calc_cam_climo: - # - true - # - true - - #cam_overwrite_climo: - # - false - # - false - - #cam_hist_loc: - # - /glade/campaign/cgd/amp/amwg/ADF_test_cases/b.e23_alpha17f.BLT1850.ne30_t232.098 - # - /glade/campaign/cgd/amp/amwg/ADF_test_cases/b.e23_alpha17f.BLT1850.ne30_t232.095 - - #cam_climo_loc: - # - /some/where/you/want/to/have/climo_files/ #MUST EDIT! - # - /the/same/or/some/other/climo/files/location - - #start_year: - # - 10 - # - 10 - - #end_year: - # - 14 - # - 14 - - #cam_ts_done: - # - false - # - false - - #cam_ts_save: - # - true - # - true - - #cam_overwrite_ts: - # - false - # - false - - #cam_ts_loc: - # - /some/where/you/want/to/have/time_series_files - # - /same/or/different/place/you/want/files - - #TEM diagnostics - #--------------- - #TEM history file number - #If missing or blank, ADF will default to h4 - #tem_hist_str: - # - cam.h4 - # - cam.h# - - #Location where TEM files are stored: - #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! - #cam_tem_loc: - # - /some/where/you/want/to/have/TEM_files/ - # - /same/or/different/place/you/want/TEM_files/ - - #Overwrite TEM files, if found? - #If set to false, then TEM creation will be skipped if files are found: - #overwrite_tem: - # - false - # - true - - #---------------------- - - -#This third set of variables provide info for the CAM baseline climatologies. -#This only matters if "compare_obs" is false: -diag_cam_baseline_climo: - - # History file list of strings to match - # eg. cam.h0 or ocn.pop.h.ecosys.nday1 or hist_str: [cam.h2,cam.h0] - # Only affects timeseries as everything else uses the created timeseries - # Default: - hist_str: cam.h0a - - #Calculate cam baseline climatologies? - #If false, the climatology files will not be created: - calc_cam_climo: true - - #Overwrite CAM climatology files? - #If false, or not present, then already existing climatology files will be skipped: - cam_overwrite_climo: false - - #Name of CAM baseline case: - cam_case_name: b.e23_alpha17f.BLT1850.ne30_t232.093 - - #Baseline case nickname - #NOTE: if nickname starts with '0' - nickname must be in quotes! - # ie '026a' as opposed to 026a - #If missing or left blank, will default to cam_case_name - case_nickname: #cool nickname - - #Location of CAM baseline history (h0) files: - #Example test files - cam_hist_loc: /glade/campaign/cgd/amp/amwg/ADF_test_cases/${diag_cam_baseline_climo.cam_case_name} - - #Location of baseline CAM climatologies: - cam_climo_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/climo - - #model year when time series files should start: - #Note: Leaving this entry blank will make time series - # start at earliest available year. - start_year: 10 - - #model year when time series files should end: - #Note: Leaving this entry blank will make time series - # end at latest available year. - end_year: 14 - - #Do time series files need to be generated? - #If True, then diagnostics assumes that model files are already time series. - #If False, or if simply not present, then diagnostics will attempt to create - #time series files from history (time-slice) files: - cam_ts_done: false - - #Save interim time series files for baseline run? - #WARNING: This can take up a significant amount of space: - cam_ts_save: true - - #Overwrite baseline time series files, if found? - #If set to false, then time series creation will be skipped if files are found: - cam_overwrite_ts: false - - #Location where time series files are (or will be) stored: - cam_ts_loc: /glade/derecho/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/ts - - #TEM diagnostics - #--------------- - #TEM history file number - #If missing or blank, ADF will default to h4 - tem_hist_str: cam.h4 - - #Location where TEM files are stored: - #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! - cam_tem_loc: /glade/derecho/scratch/${user}/${diag_cam_baseline_climo.cam_case_name}/tem/ - - #Overwrite TEM files, if found? - #If set to false, then TEM creation will be skipped if files are found: - overwrite_tem: false - - -#This fourth set of variables provides settings for calling the Climate Variability -# Diagnostics Package (CVDP). If cvdp_run is set to true the CVDP will be set up and -# run in background mode, likely completing after the ADF has completed. -# If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed -# in the diag_var_list variable listing. -# For more CVDP information: https://www.cesm.ucar.edu/working_groups/CVC/cvdp/ -diag_cvdp_info: - - # Run the CVDP on the listed run(s)? - cvdp_run: false - - # CVDP code path, sets the location of the CVDP codebase - # CGD systems path = /home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - # CISL systems path = /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - # github location = https://github.com/NCAR/CVDP-ncl - cvdp_codebase_loc: /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - - # Location where cvdp codebase will be copied to and diagnostic plots will be stored - cvdp_loc: /glade/derecho/scratch/${user}/ADF/cvdp/ - - # tar up CVDP results? - cvdp_tar: false - -# This set of variables provides settings for calling NOAA's -# Model Diagnostic Task Force (MDTF) diagnostic package. -# https://github.com/NOAA-GFDL/MDTF-diagnostics -# -# If mdtf_run: true, the MDTF will be set up and -# run in background mode, likely completing after the ADF has completed. -# -# WARNING: This currently only runs on CASPER (not derecho) -# -# The variables required depend on the diagnostics (PODs) selected. -# AMWG-developed PODS and their required variables: -# (Note that PRECT can be computed from PRECC & PRECL) -# - MJO_suite: daily PRECT, FLUT, U850, U200, V200 (all required) -# - Wheeler-Kiladis Wavenumber Frequency Spectra: daily PRECT, FLUT, U200, U850, OMEGA500 -# (will use what is available) -# - Blocking (Rich Neale): daily OMEGA500 -# - Precip Diurnal Cycle (Rich Neale): 3-hrly PRECT -# -# Many other diagnostics are available; see -# https://mdtf-diagnostics.readthedocs.io/en/main/sphinx/start_overview.html - -# -diag_mdtf_info: - # Run the MDTF on the model cases - mdtf_run: false - - # The file that will be written by ADF to input to MDTF. Call this whatever you want. - mdtf_input_settings_filename : mdtf_input.json - - ## MDTF code path, sets the location of the MDTF codebase and pre-compiled conda envs - # CHANGE if you have any: your own MDTF code, installed conda envs and/or obs_data - - mdtf_codebase_path : /glade/campaign/cgd/amp/amwg/mdtf - mdtf_codebase_loc : ${mdtf_codebase_path}/MDTF-diagnostics.v3.1.20230817.ADF - conda_root : /glade/u/apps/opt/conda - conda_env_root : ${mdtf_codebase_path}/miniconda2/envs.MDTFv3.1.20230412/ - OBS_DATA_ROOT : ${mdtf_codebase_path}/obs_data - - # SET this to a writable dir. The ADF will place ts files here for the MDTF to read (adds the casename) - MODEL_DATA_ROOT : ${diag_cam_climo.cam_ts_loc}/mdtf/inputdata/model - - # Choose diagnostics (PODs). Full list of available PODs: https://github.com/NOAA-GFDL/MDTF-diagnostics - pod_list : [ "MJO_suite" ] - - # Intermediate/output file settings - make_variab_tar: false # tar up MDTF results - save_ps : false # save postscript figures in addition to bitmaps - save_nc : false # save netCDF files of processed data (recommend true when starting with new model data) - overwrite: true # overwrite results in OUTPUT_DIR; otherwise results will be saved under a unique name - - # Settings used in debugging: - verbose : 3 # Log verbosity level. - test_mode: false # Set to true for framework test. Data is fetched but PODs are not run. - dry_run : false # Framework test. No external commands are run and no remote data is copied. Implies test_mode. - - # Settings that shouldn't change in ADF implementation for now - data_type : single_run # single_run or multi_run (only works with single right now) - data_manager : Local_File # Fetch data or it is local? - environment_manager : Conda # Manage dependencies - - - -#+++++++++++++++++++++++++++++++++++++++++++++++++++ -#These variables below only matter if you are using -#a non-standard method, or are adding your own -#diagnostic scripts. -#+++++++++++++++++++++++++++++++++++++++++++++++++++ - -#Note: If you want to pass arguments to a particular script, you can -#do it like so (using the "averaging_example" script in this case): -# - {create_climo_files: {kwargs: {clobber: true}}} - -#Name of time-averaging scripts being used to generate climatologies. -#These scripts must be located in "scripts/averaging": -time_averaging_scripts: - - create_climo_files - #- create_TEM_files #To generate TEM files, please un-comment - -#Name of regridding scripts being used. -#These scripts must be located in "scripts/regridding": -regridding_scripts: - - regrid_and_vert_interp - -#List of analysis scripts being used. -#These scripts must be located in "scripts/analysis": -analysis_scripts: - - amwg_table - - ENSO_acrossRuns - #- aerosol_gas_tables - -#List of plotting scripts being used. -#These scripts must be located in "scripts/plotting": -plotting_scripts: - - global_latlon_map - # - global_latlon_vect_map - # - zonal_mean - # - meridional_mean - # - polar_map - # - cam_taylor_diagram - # - qbo - # - ozone_diagnostics - - enso_comparison_plots - #- tape_recorder - #- tem - #- regional_map_multicase #To use this please un-comment and fill-out - #the "region_multicase" section below - -#List of CAM variables that will be processesd: -#If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed -diag_var_list: - - SWCF - - LWCF - - PRECC - - PRECL - - PSL - - Q - - U - - T - - RELHUM - - TREFHT - - TS - - TAUX - - TAUY - - FSNT - - FLNT - - LANDFRAC - - O3 - -# -# MDTF recommended variables -# - FLUT -# - OMEGA500 -# - PRECT -# - PS -# - PSL -# - U200 -# - U850 -# - V200 -# - V850 - -# Options for multi-case regional contour plots (./plotting/regional_map_multicase.py) -# region_multicase: -# region_spec: [slat, nlat, wlon, elon] -# region_time_option: # If calendar, will look for specified years. If zeroanchor will use a nyears starting from year_offset from the beginning of timeseries -# region_start_year: -# region_end_year: -# region_nyear: -# region_year_offset: -# region_month: -# region_season: -# region_variables: - -#END OF FILE diff --git a/config_for_Simone_beta05.yaml b/config_for_Simone_beta05.yaml deleted file mode 100644 index fce6d7b79..000000000 --- a/config_for_Simone_beta05.yaml +++ /dev/null @@ -1,474 +0,0 @@ -#============================== -#config_cam_baseline.yaml - -#This is the main CAM diagnostics config file -#for doing comparisons of a CAM run against -#another CAM run, or a CAM baseline simulation. - -#Currently, if one is on NCAR's Casper or -#Cheyenne machine, then only the diagnostic output -#paths are needed, at least to perform a quick test -#run (these are indicated with "MUST EDIT" comments). -#Running these diagnostics on a different machine, -#or with a different, non-example simulation, will -#require additional modifications. -# -#Config file Keywords: -#-------------------- -# -#1. Using ${xxx} will substitute that text with the -# variable referenced by xxx. For example: -# -# cam_case_name: cool_run -# cam_climo_loc: /some/where/${cam_case_name} -# -# will set "cam_climo_loc" in the diagnostics package to: -# /some/where/cool_run -# -# Please note that currently this will only work if the -# variable only exists in one location in the file. -# -#2. Using ${.xxx} will do the same as -# keyword 1 above, but specifies which sub-section the -# variable is coming from, which is necessary for variables -# that are repeated in different subsections. For example: -# -# diag_basic_info: -# cam_climo_loc: /some/where/${diag_cam_climo.start_year} -# -# diag_cam_climo: -# start_year: 1850 -# -# will set "cam_climo_loc" in the diagnostics package to: -# /some/where/1850 -# -#Finally, please note that for both 1 and 2 the keywords must be lowercase. -#This is because future developments will hopefully use other keywords -#that are uppercase. Also please avoid using periods (".") in variable -#names, as this will likely cause issues with the current file parsing -#system. -#-------------------- -# -##============================== - -#diag_loc: /glade/scratch/richling/adf-output/f.cam6_3_132.FMTHIST_ne30.taubgnd5.energy_front_off_rd_beta_1_vs_f.cam6_3_119.FMTHIST_ne30.r328_gamma0.33_soae.nudged_dst11.001/ - -#climo_loc: /glade/scratch/richling/adf-output/ADF-data/climos/ #/glade/campaign/cgd/amp/amwg/climo/ -#ts_loc: /glade/scratch/richling/adf-output/ADF-data/timeseries/ - -user: 'behroozr' - -#This first set of variables specify basic info used by all diagnostic runs: -diag_basic_info: - # diag_loc: /glade/derecho/scratch/richling/adf-output/${diag_cam_climo.cam_case_name}_vs_${diag_cam_baseline_climo.cam_case_name}/ - - # climo_loc: /glade/derecho/scratch/richling/adf-output/ADF-data/climo/ - # ts_loc: /glade/derecho/scratch/richling/adf-output/ADF-data/timeseries/ - - diag_loc: /glade/derecho/scratch/behroozr/Budgets/output_test/${diag_cam_climo.cam_case_name}_vs_${diag_cam_baseline_climo.cam_case_name}/ - - climo_loc: /glade/derecho/scratch/behroozr/Budgets//output_test/ADF-data/climo/ - ts_loc: /glade/derecho/scratch/behroozr/Budgets/output_test/ADF-data/timeseries/ - - - #History file string to match (eg. cam.h0 or ocn.pop.h.ecosys.nday1) - # Only affects timeseries as everything else uses timeseries - # Leave off trailing '.' - #Default: cam.h0 - #hist_str: cam.h0a - - #Is this a model vs observations comparison? - #If "false" or missing, then a model-model comparison is assumed: - compare_obs: false - - #Generate HTML website (assumed false if missing): - #Note: The website files themselves will be located in the path - #specified by "cam_diag_plot_loc", under the "/website" subdirectory, - #where "" is the subdirectory created for this particular diagnostics run - #(usually "case_vs_obs_XXX" or "case_vs_baseline_XXX"). - create_html: true - - #Location of observational datasets: - #Note: this only matters if "compare_obs" is true and the path - #isn't specified in the variable defaults file. - obs_data_loc: /glade/campaign/cgd/amp/amwg/ADF_obs - - #Location where re-gridded CAM climatology files are stored: - cam_regrid_loc: ${diag_loc}regrid/ - - #Overwrite CAM re-gridded files? - #If false, or missing, then regridding will be skipped for regridded variables - #that already exist in "cam_regrid_loc": - cam_overwrite_regrid: false - - #Location where diagnostic plots are stored: - cam_diag_plot_loc: ${diag_loc}diag-plot/ - - #Use default variable plot settings? - #If "true", then variable-specific plotting attributes as defined in - #ADF/lib/adf_variable_defaults.yaml will be used: - use_defaults: true - - #Location of ADF variable plotting defaults YAML file - #if not using the one in ADF/lib: - #defaults_file: /some/path/to/defaults/file - - #Vertical pressure levels (in hPa) on which to plot 3-D variables - #when using horizontal (e.g. lat/lon) map projections. - #If this config option is missing, then no 3-D variables will be plotted on - #horizontal maps: - plot_press_levels: [200,850] - - #Apply monthly weights to seasonal averages. - #If False or missing, then all months are - #given the same weight: - weight_season: True - - num_procs: 8 - - redo_plot: false - - - - - - - - - -#This second set of variables provides info for the CAM simulation(s) being diagnosed: -diag_cam_climo: - - #Calculate climatologies? - #If false, neither the climatology or time-series files will be created: - calc_cam_climo: true - - #Overwrite CAM climatology files? - #If false, or not prsent, then already existing climatology files will be skipped: - cam_overwrite_climo: false - - #Name of CAM case (or CAM run name): - #cam_case_name: f.cam6_3_153.FCMTnudged_climate_chemistry_ne30.factor_fix - # case_nickname: f.cam6_3_153.FCMTnudged_climate_chemistry_ne30.factor_fix - - #cam_case_name: f.cam6_3_160.FCMT_ne30.moving_mtn.002 - #case_nickname: f.cam6_3_160.FCMT_ne30.moving_mtn.002 - #start_year: 1996 - #end_year: 1997 - - hist_str: cam.h0a - #hist_str: cam.h0 - - - # cam_case_name: FCnudged_f09.mam.Jul9.1995_2020.001 - # case_nickname: FCnudged_f09.mam.Jul9.1995_2020.001 - # start_year: 2002 - # end_year: 2019 - - - # cam_case_name: FCnudged_f09.mam.mar27.2000_2021.001 - # case_nickname: FCnudged_f09.mam.mar27.2000_2021.001 - # start_year: 2002 - # end_year: 2019 - - # cam_case_name: b.e22.BWSSP585cmip6ctsslh.f09_f09_mg17.cesm2.2_vsl03.present.CMIP_MLtracers_tagged.Halogensv002 - # case_nickname: B_585P_ML - # start_year: 2016 - # end_year: 2025 - - # cam_case_name: b.e22.BWSSP585cmip6ctsslh.f09_f09_mg17.cesm2.2_vsl03.future.CMIP_MLtracers_tagged.Halogensv002 - # case_nickname: B_585F_ML - # start_year: 2090 - # end_year: 2099 - - # cam_case_name: b.e22.BWSSP585cmip6ctsslh.f09_f09_mg17.cesm2.2_vsl03.future.CMIP_MLtracers_tagged.Halogensv001 - # case_nickname: B_585F_Ord - # start_year: 2090 - # end_year: 2099 - - cam_case_name: b.e22.BWSSP585cmip6ctsslh.f09_f09_mg17.cesm2.2_vsl03.future.CMIP_MLtracers_tagged.Halogensv001 - case_nickname: B_585P_Ord - start_year: 2016 - end_year: 2025 - - cam_case_name: f.e30_beta05.FCts4MTHIST.ne30_L93.cmip7.001 - case_nickname: beta_05 - start_year: 1980 - end_year: 1980 - - # cam_case_name: f.e22.FCnudged.f09_f09_mg17.slh.2000.fire.ct.finn.mosaic_cams6.001 - # case_nickname: Ben_FV - # start_year: 2003 - # end_year: 2003 - - # cam_case_name: f.e30_alpha04a.FCts4MTHIST.ne30_L93.cmip7_old_volc - # case_nickname: beta_04 - # start_year: 1980 - # end_year: 1980 - - # #cam_case_name: f.cam6_3_160.FMTHIST_ne30.moving_mtn.output.001 - # #case_nickname: f.cam6_3_160.FMTHIST_ne30.moving_mtn.output.001 - # #start_year: 1996 - # #end_year: 2001 - - - - - - # #Location of CAM history (h0) files: - # #/glade/derecho/scratch/tilmes/archive/f.cam6_3_153.FCMTnudged_climate_chemistry_ne30.001/atm/hist - # #cam_hist_loc: /glade/derecho/scratch/tilmes/archive/${diag_cam_climo.cam_case_name}/atm/hist/ - #cam_hist_loc: /glade/campaign/acom/acom-climate/UTLS/shawnh/archive/${diag_cam_climo.cam_case_name}/atm/hist/ - #cam_hist_loc: /glade/campaign/acom/acom-weather/behroozr/VSLS/cases2024//${diag_cam_climo.cam_case_name}/atm/hist/ - cam_hist_loc: /glade/derecho/scratch/shawnh/archive//${diag_cam_climo.cam_case_name}/atm/hist/ - #cam_hist_loc: /glade/campaign/acom/acom-climate/UTLS/shawnh/archive/${diag_cam_climo.cam_case_name}/atm/hist/ - #cam_hist_loc: /glade/campaign/acom/acom-da/Methane_simulations/f.e22.FCnudged.f09_f09_mg17.slh.2000.fire.ct.finn.mosaic_cams6.001/H0/ - - # cam_case_name: f.e22.FHIST.ne0np4.India07.ne30x1_ne30x1_mt12_cesm2.2_rel_bugFixed_noPATC - # case_nickname: f.e22.FHIST.ne0np4.India07.ne30x1_ne30x1_mt12_cesm2.2_rel_bugFixed_noPATC - # start_year: 2002 - # end_year: 2002 - # cam_hist_loc: /glade/campaign/acom/acom-weather/behroozr/India_Dust/Simulation2_${diag_cam_climo.cam_case_name}/atm/hist/ - - - #Location of CAM climatologies: - cam_climo_loc: ${climo_loc}${diag_cam_climo.cam_case_name}/yrs_1995_2001/ - - #model year when time series files should start: - #Note: Leaving this entry blank will make time series - # start at earliest available year. - #start_year: 2001 - - #model year when time series files should end: - #Note: Leaving this entry blank will make time series - # end at latest available year. - #end_year: 2001 - - #Do time series files need to be generated? - #If True, then diagnostics assumes that model files are already time series. - #If False, or if simply not present, then diagnostics will attempt to create - #time series files from history (time-slice) files: - cam_ts_done: false - - #Save interim time series files? - #WARNING: This can take up a significant amount of space: - cam_ts_save: true - - #Overwrite time series files, if found? - #If set to false, then time series creation will be skipped if files are found: - cam_overwrite_ts: false - - #Location where time series files are (or will be) stored: - cam_ts_loc: ${ts_loc}${diag_cam_climo.cam_case_name}/yrs_1995_2001/ - - #TEM diagnostics - #--------------- - #TEM history file number - #If missing or blank, ADF will default to h4 - tem_hist_str: cam.h4 - - #Location where TEM files are stored: - #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped! - cam_tem_loc: /glade/scratch/${user}/${diag_cam_climo.cam_case_name}/tem/ - - #Overwrite TEM files, if found? - #If set to false, then TEM creation will be skipped if files are found: - overwrite_tem: false - - -#This third set of variables provide info for the CAM baseline climatologies. -#This only matters if "compare_obs" is false: -diag_cam_baseline_climo: - - #Calculate cam baseline climatologies? - #If false, neither the climatology or time-series files will be created: - calc_cam_climo: true - - #Overwrite CAM climatology files? - #If false, or not present, then already existing climatology files will be skipped: - cam_overwrite_climo: false - #hist_str: cam.h0a - hist_str: cam.h0 - - #Name of CAM baseline case: - cam_case_name: FCnudged_f09.mam.mar27.2000_2021.001 - - case_nickname: FCnudged_f09.mam.mar27.2000_2021.001 - - #Location of CAM baseline history (h0) files: - #/glade/derecho/scratch/tilmes/archive/f.cam6_3_153.FCMTnudged_ne30.001/atm/hist - #cam_hist_loc: /glade/derecho/scratch/tilmes/archive/${diag_cam_baseline_climo.cam_case_name}/atm/hist/ - cam_hist_loc: /glade/campaign/acom/acom-climate/UTLS/shawnh/archive/${diag_cam_baseline_climo.cam_case_name}/atm/hist/ - - - #Location of baseline CAM climatologies: - cam_climo_loc: ${climo_loc}${diag_cam_baseline_climo.cam_case_name}/yrs_1995_2001/ - - #model year when time series files should start: - #Note: Leaving this entry blank will make time series - # start at earliest available year. - start_year: 2002 - - #model year when time series files should end: - #Note: Leaving this entry blank will make time series - # end at latest available year. - end_year: 2002 - - #Do time series files need to be generated? - #If True, then diagnostics assumes that model files are already time series. - #If False, or if simply not present, then diagnostics will attempt to create - #time series files from history (time-slice) files: - cam_ts_done: false - - #Save interim time series files for baseline run? - #WARNING: This can take up a significant amount of space: - cam_ts_save: true - - #Overwrite baseline time series files, if found? - #If set to false, then time series creation will be skipped if files are found: - cam_overwrite_ts: false - - #Location where time series files are (or will be) stored: - cam_ts_loc: ${ts_loc}${diag_cam_baseline_climo.cam_case_name}/yrs_1995_2001/ - - -#This fourth set of variables provides settings for calling the Climate Variability -# Diagnostics Package (CVDP). If cvdp_run is set to true the CVDP will be set up and -# run in background mode, likely completing after the ADF has completed. -# If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed -# in the diag_var_list variable listing. -# For more CVDP information: https://www.cesm.ucar.edu/working_groups/CVC/cvdp/ -diag_cvdp_info: - - # Run the CVDP on the listed run(s)? - cvdp_run: false - - # CVDP code path, sets the location of the CVDP codebase - # CGD systems path = /home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - # CISL systems path = /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - # github location = https://github.com/NCAR/CVDP-ncl - cvdp_codebase_loc: /glade/u/home/asphilli/CESM-diagnostics/CVDP/Release/v5.2.0/ - - # Location where cvdp codebase will be copied to and diagnostic plots will be stored - cvdp_loc: ${diag_loc} - - # tar up CVDP results? - #cvdp_tar: false - - -#+++++++++++++++++++++++++++++++++++++++++++++++++++ -#These variables below only matter if you are using -#a non-standard method, or are adding your own -#diagnostic scripts. -#+++++++++++++++++++++++++++++++++++++++++++++++++++ - -#Note: If you want to pass arguments to a particular script, you can -#do it like so (using the "averaging_example" script in this case): -# - {averaging_example: {kwargs: {clobber: true}}} - -#Name of time-averaging scripts being used to generate climatologies. -#These scripts must be located in "scripts/averaging": -time_averaging_scripts: - - create_climo_files - #- create_TEM_files - -#Name of regridding scripts being used. -#These scripts must be located in "scripts/regridding": -regridding_scripts: - - regrid_and_vert_interp - -#List of analysis scripts being used. -#These scripts must be located in "scripts/analysis": -analysis_scripts: - - amwg_table - -#List of plotting scripts being used. -#These scripts must be located in "scripts/plotting": -plotting_scripts: - - global_latlon_map - - zonal_mean - - meridional_mean - - polar_map - - global_latlon_vect_map - - cam_taylor_diagram - - qbo - #- seasonal_cycle - #- tem - - tape_recorder - -#List of CAM variables that will be processesd: -#If CVDP is to be run PSL, TREFHT, TS and PRECT (or PRECC and PRECL) should be listed -diag_var_list: - - AODDUSTdn - - AODVISdn - - CLDHGH - - CLDICE - - CLDLIQ - - CLDLOW - - CLDMED - - CLDTOT - - CLOUD - - FLNS - - FLNT - - FLNTC - - FSNS - - FSNT - - FSNTC - - LHFLX - - LWCF - - OMEGA500 - - PBLH - - PRECL - - PRECT - - PRECSL - - PRECSC - - PRECC - - PS - - PSL - - QFLX - - Q - - RELHUM - - SHFLX - - SST - - SWCF - - T - - TAUX - - TAUY - - TGCLDIWP - - TGCLDLWP - - TMQ - - TREFHT - - TS - - U - - U10 - - ICEFRAC - - OCNFRAC - - LANDFRAC - - RESTOM - - BC - - POM - - SO4 - - SOA - - NH4HSO4 - - DUST - - SeaSalt - - - - -# - -# Options for TEM diagnostics (./averaging/create_TEM_files.py and ./plotting/temp.py) -tem_info: - #Location where TEM files are stored: - #If path not specified or commented out, skip TEM calculation - #If no path, diagnostics wont run even if declared in averaging/plotting scripts below - tem_loc: /glade/scratch/richling/adf-output/ADF-data/TEM/ - - #TEM history file number - hist_num: h4 - - overwrite_tem_case: false - overwrite_tem_base: false - -#END OF FILE diff --git a/scripts/analysis/aerosol_gas_tables_Tropopause_version0.py b/scripts/analysis/aerosol_gas_tables_Tropopause_version0.py deleted file mode 100644 index e8759678c..000000000 --- a/scripts/analysis/aerosol_gas_tables_Tropopause_version0.py +++ /dev/null @@ -1,1390 +0,0 @@ -import numpy as np -import xarray as xr -import sys -from pathlib import Path -import warnings # use to warn user about missing files. - -from datetime import datetime -import numpy as np -import itertools - -try: - import pandas as pd -except ImportError: - print("Pandas module does not exist in python path, but is needed for amwg_table.") - print("Please install module, e.g. 'pip install pandas'.") - sys.exit(1) -#End except - -# Import necessary ADF modules: -from adf_base import AdfError - -def aerosol_gas_tables(adfobj): - ''' - Calculate aerosol and gaseous budget tables - - Default set of variables: change in lib/adf_variable_defaults.yaml - ------------------------- - GAS_VARIABLES: ['CH4','CH3CCL3', 'CO', 'O3', 'ISOP', 'MTERP', 'CH3OH', 'CH3COCH3'] - AEROSOL_VARIABLES: ['AOD','SOA', 'SALT', 'DUST', 'POM', 'BC', 'SO4'] - - Default output for tables: - - Gases: - ------ - CH4_BURDEN (Tg), CH4_CHEM_LOSS (Tg/yr), CH4_LIFETIME (years) - - CH3CCL3_BURDEN (Tg), CH3CCL3_CHEM_LOSS (Tg/yr), CH3CCL3_LIFETIME (days) - - CO_EMIS (Tg/yr), CO_BURDEN (Tg), CO_CHEM_LOSS (Tg/yr), CO_CHEM_PROD (Tg/yr), CO_DRYDEP (Tg/yr) - CO_TDEP (Tg/yr), CO_LIFETIME (days), CO_TEND (Tg/yr) - - O3_BURDEN (Tg), O3_CHEM_LOSS (Tg/yr), O3_CHEM_PROD (Tg/yr), O3_DRYDEP (Tg/yr), O3_TDEP (Tg/yr) - O3_LIFETIME (days), O3_TEND (Tg/yr), O3_STE (Tg/yr) - - LNOx_PROD (Tg N/yr) - - ISOP_EMIS (Tg/yr), ISOP_BURDEN (Tg) - - Monoterpene_EMIS (Tg/yr), Monoterpene_BURDEN (Tg) - - Methanol_EMIS (Tg/yr), Methanol_BURDEN (Tg), Methanol_DRYDEP (Tg/yr), Methanol_WETDEP (Tg/yr), Methanol_TDEP (Tg/yr) - - Acetone_EMIS (Tg/yr), Acetone_BURDEN (Tg), Acetone_DRYDEP (Tg/yr), Acetone_WETDEP (Tg/yr), Acetone_TDEP (Tg/yr) - - - - Aerosols: - --------- - AOD_mean - - SOA_BURDEN (Tg), SOA_CHEM_LOSS (Tg/yr), SOA_DRYDEP (Tg/yr), SOA_WETDEP (Tg/yr), SOA_GAEX (Tg/yr), SOA_LIFETIME (days) - - SALT_EMIS (Tg/yr), SALT_BURDEN (Tg), SALT_DRYDEP (Tg/yr), SALT_WETDEP (Tg/yr), SALT_LIFETIME (days) - - DUST_EMIS (Tg/yr), DUST_BURDEN (Tg), DUST_DRYDEP (Tg/yr), DUST_WETDEP (Tg/yr), DUST_LIFETIME (days) - - POM_EMIS (Tg/yr), POM_BURDEN (Tg), POM_DRYDEP (Tg/yr), POM_WETDEP (Tg/yr), POM_LIFETIME (days) - - BC_EMIS (Tg/yr), BC_BURDEN (Tg), BC_DRYDEP (Tg/yr), BC_WETDEP (Tg/yr), BC_LIFETIME (days) - - SO4_EMIS_elevated (Tg S/yr), SO4_BURDEN (Tg S), SO4_DRYDEP (Tg S/yr), SO4_WETDEP (Tg S/yr), SO4_GAEX (Tg S/yr) - SO4_LIFETIME (days), SO4_AQUEOUS (Tg S/yr), SO4_NUCLEATION (Tg S/yr) - - - List of variable names and descriptions for clarity - --------------------------------------------------- - - ListVars: list of all available variables from given history file - - GAS_VARIABLES: list fo necessary CAM gaseous variables - - AEROSOL_VARIABLES: list fo necessary CAM aerosol variables - - AEROSOLS: list of necessary aerosols for computations - - - MODIFICATION HISTORY: - Behrooz Roozitalab, 02, NOV, 2022: VERSION 1.00 - - Initial version - - Justin Richling, 27 Nov, 2023 - - updated to fit to ADF and check with old AMWG chem/aerosol tables - - fixed: - * added difference bewtween cases column to tables - - Behrooz Roozitalab, 8 Aug, 2024 - - fixed: - * lifetime inconsitencies - * Removed redundant calculations to improve the speed - * Verified the results against the NCL script. - - Behrooz Roozitalab, 5 Jun, 2025 - - fixed: - * Fix the bugs in the calculation (when converting from Jupyterhub to ADF) - * add the 'U' variable in dic_SE - * make the code faster by modifying make_Dic_scn_var_comp - * Add a condition to calculate whole world budgets when O3 is not find. - * Update pressure calculation in a more general way. - - Behrooz Roozitalab, 20 Aug, 2025 _ Version 0 - - fixed: - * the html page was not created, it is fixed. - * added "hm" as a case to enable using annual averaged files in addition to monthly files. - * New method of defining troposphere, use ozone (150ppb) or Trop_P. If not found, calculate total column - * Added DMS to gases list - reported as DMS not S - * Automatic addition of gaseous compounds even when not defined in the default list, - * based on Carbon MW (12). It still needs ADF modification to read a list from yaml file. - ''' - - - #Notify user that script has started: - msg = "\n Calculating chemistry/aerosol budget tables..." - print(f"{msg}\n {'-' * (len(msg)-3)}") - - # Inputs - #------- - # Variable defaults info - res = adfobj.variable_defaults # dict of variable-specific plot preferences - bres = res['budget_tables'] - # list of the gaseous variables to be caculated. - GAS_VARIABLES = bres['GAS_VARIABLES'] - - # list of the aerosol variables to be caculated. - AEROSOL_VARIABLES = bres['AEROSOL_VARIABLES'] - - #list of all the variables to be caculated. - VARIABLES = GAS_VARIABLES + AEROSOL_VARIABLES - - # For the case that outputs are saved for a specific region. - # i.e., when using fincllonlat in user_nl_cam - ext1_SE = bres['ext1_SE'] - - # Tropospheric Values - # ------------------- - # if True, calculate only Tropospheric values - # if False, all layers - # tropopause is defiend as either directly or indirectly. Look for tropopause to see the definition - Tropospheric = bres['Tropospheric'] - - ### NOT WORKING FOR NOW - # To calculate the budgets only for a region - # Lat/Lon extent - limit = bres['limit'] - regional = bres['regional'] - - # Dictionary for Molecular weights. Keys must be consistent with variable name - # For aerosols, the MW is used only for chemical loss, chemical production, and elevated emission calculations - # For SO4, we report everything in terms of Sulfur, so we use Sulfur MW here - MW = bres['MW'] - - # automatic generation of MW - for var in VARIABLES: - if var not in MW.keys(): - print(f"using Carbon molecular weight for {var}") - MW[var]=12 - - - # Avogadro's Number - AVO = float(bres['AVO']) - # gravity - gr = float(bres['gr']) - # Mw air - Mwair = float(bres['Mwair']) - - # The variables in the list below must be aerosols - do not add AOD and DAOD - # no need to change this list, unless for a specific need! - AEROSOLS = bres['AEROSOLS'] - - # Start gathering case, path, and data info - #----------------------------------------- - - # CAM simulation variables (these quantities are always lists): - case_names = adfobj.get_cam_info('cam_case_name', required=True) - - # Grab all case nickname(s) - test_nicknames_list = adfobj.case_nicknames["test_nicknames"] - nicknames_list = test_nicknames_list - # Grab climo years - start_years = adfobj.climo_yrs["syears"] - end_years = adfobj.climo_yrs["eyears"] - - #Grab history strings: - hist_strs = adfobj.hist_string["test_hist_str"] - - # Grab history file locations from config yaml file - hist_locs = adfobj.get_cam_info("cam_hist_loc", required=True) - - # Check if this is test model vs baseline model - # If so, update test case(s) lists created above - if not adfobj.compare_obs: - # Get baseline case info - case_names += [adfobj.get_baseline_info("cam_case_name")] - nicknames_list += [adfobj.case_nicknames["base_nickname"]] - - # Grab climo years - start_years += [adfobj.climo_yrs["syear_baseline"]] - end_years += [adfobj.climo_yrs["eyear_baseline"]] - - # Get history file info - hist_strs += [adfobj.hist_string["base_hist_str"]] - hist_locs += [adfobj.get_baseline_info("cam_hist_loc")] - # End if - - # Check to ensure number of case names matches number history file locations. - # If not, exit script - if len(hist_locs) != len(case_names): - errmsg = "Error: number of cases does not match number of history file locations. Script is exiting." - raise AdfError(errmsg) - - # Initialize nicknames dictionary - #nicknames = {} - - # Filter the list to include only strings that are possible h0 strings - # - Search for either h0 or h0a - substrings = {"cam.h0","cam.h0a","cam.hm"} - case_hist_strs = [] - for cam_case_str in hist_strs: - # Check each possible h0 string - for string in cam_case_str: - if string in substrings: - case_hist_strs.append(string) - break - - # Create path object for the CAM history file(s) location: - data_dirs = [] - for case_idx,case in enumerate(nicknames_list): - - print(f"\t Looking for history location: {hist_locs[case_idx]}") - - - #Check that history file input directory actually exists: - if (hist_locs[case_idx] is None) or (not Path(hist_locs[case_idx]).is_dir()): - errmsg = f"History files directory '{hist_locs[case_idx]}' not found. Script is exiting." - raise AdfError(errmsg) - - #Write to debug log if enabled: - adfobj.debug_log(f"DEBUG: location of history files is {str(hist_locs[case_idx])}") - # Update list for found directories - data_dirs.append(hist_locs[case_idx]) - - # End gathering case, path, and data info - #----------------------------------------- - # Periods of Interest - # ------------------- - # choose the period of interest. Plots will be averaged within this period - durations = {} - num_yrs = {} - - # Main function - #-------------- - # Set dictionary of components for each case - Dic_scn_var_comp = {} - areas = {} - trops = {} - insides = {} - for i,case in enumerate(nicknames_list): - - start_year = start_years[i] - end_year = end_years[i] + 1 - start_date = f"{start_year}-1-1" - end_date = f"{end_year}-1-1" - - # Create time periods - start_period = datetime.strptime(start_date, "%Y-%m-%d") - end_period = datetime.strptime(end_date, "%Y-%m-%d") - - # Calculated duration of time period in seconds? - durations[case] = (end_period-start_period).days*86400 #+365*86400 - - - # Get number of years for calculations - num_yrs[case] = (int(end_year)-int(start_year)) #+1 - - # Get currenty history file directory - data_dir = data_dirs[i] - - # Get all files, lats, lons, and area weights for current case - Files,Lats,Lons,areas[case],ext1_SE = Get_files(adfobj,data_dir,start_year,end_year,case_hist_strs[i],area=True) - # find the name of all the variables in the file. - # this will help the code to work for the variables that are not in the files (assingn 0s) - tmp_file = xr.open_dataset(Path(data_dir) / Files[0]) - ListVars = list(tmp_file.variables) - - # Set up and fill dictionaries for components for current cases - dic_SE = set_dic_SE(ListVars,ext1_SE,VARIABLES) - dic_SE = fill_dic_SE(adfobj, dic_SE, VARIABLES, ListVars, ext1_SE, AEROSOLS, MW, AVO, gr, Mwair) - - text = f'\n\t Calculating values for {case}' - print(text) - print("\t " + "-" * (len(text) - 2)) - - # Gather dictionary data for current case - # NOTE: The calculations can take a long time... - Dic_crit, Dic_scn_var_comp[case],Tropospheric,tropospheric_method = make_Dic_scn_var_comp(adfobj, VARIABLES, data_dir, dic_SE, Files, ext1_SE, AEROSOLS,Tropospheric) - # Regional refinement - # NOTE: This function 'Inside_SE' is unavailable at the moment! - JR 10/2024 - if regional: - #inside = Inside_SE_region(current_lat,current_lon,dir_shapefile) - inside = Inside_SE(Lats,Lons,limit) - else: - if len(np.shape(areas[case])) == 1: - inside = np.full((len(Lons)),True) - else: - inside = np.full((len(Lats),len(Lons)),True) - - # Set critical threshold - current_crit = Dic_crit - if Tropospheric: - if tropospheric_method=='ozone': - # using ozone <150 ppb - trop = np.where(current_crit>150,np.nan,current_crit) - elif tropospheric_method=='tropopause': - # using pressure > tropopause pressure - trop = np.where(current_crit['Pressure']150,current_crit,np.nan) - else: - trop=current_crit - trops[case] = trop - insides[case] = inside - - # Make and save tables - table_kwargs = {"adfobj":adfobj, - "Dic_scn_var_comp":Dic_scn_var_comp, - "areas":areas, - "trops":trops, - "case_names":case_names, - "nicknames":nicknames_list, - "durations":durations, - "insides":insides, - "num_yrs":num_yrs, - "AEROSOLS":AEROSOLS} - - #print(table_kwargs) - - # Create the budget tables - #------------------------- - # Aerosols - if len(AEROSOL_VARIABLES) > 0: - print("\tMaking table for aerosols") - make_table(vars=AEROSOL_VARIABLES, chem_type='aerosols', **table_kwargs) - # Gases - if len(GAS_VARIABLES) > 0: - print("\tMaking table for gases") - make_table(vars=GAS_VARIABLES, chem_type='gases', **table_kwargs) -####### - -################## -# Helper functions -################## - -def list_files(adfobj, directory, start_year ,end_year, h_case): - - """ - This function extracts the files in the directory that are within the chosen dates - and history number. - """ - - # History file year range - yrs = np.arange(int(start_year), int(end_year)) - - all_filenames = [] - for i in yrs: -# all_filenames.append(sorted(Path(directory).glob(f'*.{h_case}.{i}-*'))) - all_filenames.append(sorted(Path(directory).glob(f'*.{h_case}.{i}*'))) - - #print(directory) - # Flattening the list of lists - filenames = list(itertools.chain.from_iterable(sorted(all_filenames))) - if len(filenames)==0: - #sys.exit(" Directory has no outputs ") - msg = f"chem/aerosol tables, 'list_files':" - msg += f"\n\t - Directory '{directory}' has no outputs." - adfobj.debug_log(msg) - - return filenames -##### - - -def Get_files(adfobj, data_dir, start_year, end_year, h_case, **kwargs): - - """ - This function retrieves the files, latitude, and longitude information - in all the directories within the chosen dates. - """ - ext1_SE = kwargs.pop('ext1_SE','') - area = kwargs.pop('area',False) - - Earth_rad=6.371e6 # Earth Radius in meters - - current_files = list_files(adfobj, data_dir, start_year, end_year,h_case) - # get the Lat and Lons for each case - tmp_file = xr.open_dataset(Path(data_dir) / current_files[0]) - lon = tmp_file['lon'+ext1_SE].data - lon[lon > 180.] -= 360 # shift longitude from 0-360˚ to -180-180˚ - lat = tmp_file['lat'+ext1_SE].data - - if area == True: - try: - tmp_area = tmp_file['area'+ext1_SE].data - Earth_area = 4 * np.pi * Earth_rad**(2) - - areas = tmp_area*Earth_area/np.nansum(tmp_area) - except KeyError: - try: - tmp_area = tmp_file['AREA'+ext1_SE].isel(time=0).data - areas=tmp_area - #Earth_area = 4 * np.pi * Earth_rad**(2) - #areas = tmp_area*Earth_area/np.nansum(tmp_area) - except: - dlon = np.abs(lon[1]-lon[0]) - dlat = np.abs(lat[1]-lat[0]) - - lon2d,lat2d = np.meshgrid(lon,lat) - #area=np.zeros_like(lat2d) - - dy = Earth_rad*dlat*np.pi/180 - dx = Earth_rad*np.cos(lat2d*np.pi/180)*dlon*np.pi/180 - - tmp_area = dx*dy - areas = tmp_area - # End if - - # Variables to return - return current_files,lat,lon,areas,ext1_SE -##### - -def set_dic_SE(ListVars, ext1_SE,variables): - """ - Initialize dictionary to house all the relevant tabel data - """ - - # Initialize dictionary - #---------------------- - dic_SE={} - - # Chemistry - #---------- - dic_SE['U']={'U'+ext1_SE:1} - dic_SE['O3']={'O3'+ext1_SE:1e9} # covert to ppb for Tropopause calculation - dic_SE['CH4']={'CH4'+ext1_SE:1} - dic_SE['CO']={'CO'+ext1_SE:1} - - dic_SE['ISOP']={'ISOP'+ext1_SE:1} - dic_SE['MTERP']={'MTERP'+ext1_SE:1} - dic_SE['CH3OH']={'CH3OH'+ext1_SE:1} - dic_SE['CH3COCH3']={'CH3COCH3'+ext1_SE:1} - dic_SE['CH3CCL3']={'CH3CCL3'+ext1_SE:1} - dic_SE['CHBR3']={'CHBR3'+ext1_SE:1} - dic_SE['CH2BR2']={'CH2BR2'+ext1_SE:1} - - # Aerosols - #--------- - - dic_SE['DAOD']={'AODDUSTdn'+ext1_SE:1} - dic_SE['AOD']={'AODVISdn'+ext1_SE:1} - - dic_SE['DUST']={'dst_a1'+ext1_SE:1, - 'dst_a2'+ext1_SE:1, - 'dst_a3'+ext1_SE:1} - - dic_SE['SALT']={'ncl_a1'+ext1_SE:1, - 'ncl_a2'+ext1_SE:1, - 'ncl_a3'+ext1_SE:1} - - dic_SE['POM']={'pom_a1'+ext1_SE:1, - 'pom_a4'+ext1_SE:1} - - dic_SE['BC']={'bc_a1'+ext1_SE:1, - 'bc_a4'+ext1_SE:1} - - - dic_SE['SO4']={'so4_a1'+ext1_SE:1, - 'so4_a2'+ext1_SE:1, - 'so4_a3'+ext1_SE:1, - 'so4_a5'+ext1_SE:1} - - # FOR SOA, first check if the integrated bins are included - if (('soa_a1'+ext1_SE in ListVars ) & ('soa_a1'+ext1_SE in ListVars )): - dic_SE['SOA'] = {'soa_a1'+ext1_SE:1, - 'soa_a2'+ext1_SE:1} - else: - dic_SE['SOA'] = {'soa1_a1'+ext1_SE:1, - 'soa2_a1'+ext1_SE:1, - 'soa3_a1'+ext1_SE:1, - 'soa4_a1'+ext1_SE:1, - 'soa5_a1'+ext1_SE:1, - 'soa1_a2'+ext1_SE:1, - 'soa2_a2'+ext1_SE:1, - 'soa3_a2'+ext1_SE:1, - 'soa4_a2'+ext1_SE:1, - 'soa5_a2'+ext1_SE:1} - - dic_SE['DMS']={'DMS'+ext1_SE:1} - #dic_SE['TROP_P']={'TROP_P'+ext1_SE:1} - - - # automatic generation of dic_SE - for var in variables: - if var not in dic_SE.keys(): - dic_SE[var]={var+ext1_SE:1} - - # consider for OASISS DMS separately - if var=='DMS': - dic_SE['DMS_OASISS']={'DMS_OASISS'+ext1_SE:1} - # End if - - return dic_SE -##### - -def fill_dic_SE(adfobj, dic_SE, variables, ListVars, ext1_SE, AEROSOLS, MW, AVO, gr, Mwair): - """ - Function for dealing with conversion factors for different components and filling the main data - dictionary 'dic_SE' - - Input dictionary and return updated dictionary 'dic_SE' - - Arguments - --------- - variables : list - - list of main variables? - ListVars : list - - list of ??????? - - Returns - ------- - dic_SE : dict - - full dictionary of derived variables - - Some conversion factors need density or Layer's pressure, that will be accounted for when reading the files. - We convert everying to kg/m2/s or kg/m2 or kg/s, so that final Tg/yr or Tg results are consistent - """ - - # Logging info message - msg = f"chem/aerosol tables: 'fill_dic_SE'" - - for var in variables: - - if 'AOD' in var: - dic_SE[var+'_AOD']={} - else: - dic_SE[var+'_BURDEN']={} - dic_SE[var+'_CHML']={} - dic_SE[var+'_CHMP']={} - - dic_SE[var+'_SF']={} - dic_SE[var+'_CLXF']={} - - dic_SE[var+'_DDF']={} - dic_SE[var+'_WDF']={} - - if var in AEROSOLS: - dic_SE[var+'_GAEX']={} - dic_SE[var+'_DDFC']={} - dic_SE[var+'_WDFC']={} - else: - dic_SE[var+'_TEND']={} - dic_SE[var+'_LNO']={} - # End if - - # We have nucleation and aqueous chemistry for sulfate. - if var=='SO4': - dic_SE[var+'_NUCL']={} - dic_SE[var+'_AQS']={} - # End if - - # Grab the variable keys - var_keys = dic_SE[var].keys() - - for key in var_keys: - msg += f"\n\t Creating component of {var}: {key}" - - # for CHML and CHMP: - # original unit : [molec/cm3/s] - # following Tilmes code to convert to [kg/m2/s] - # conversion: Mw*rho*delP*1e3/Avo/gr - # rho and delP will be applied when reading the files in SEbudget function. - - # for AOD and DAOD: - if 'AOD' in var: - if key in ListVars: - dic_SE[var+'_AOD'][key+ext1_SE]=1 - else: - dic_SE[var+'_AOD']['PS'+ext1_SE]=0. - # End if - continue # AOD doesn't need any other budget calculations - # End if - - # for CHML and CHMP: - # original unit : [molec/cm3/s] - # following Tilmes code to convert to [kg/m2/s] - # conversion: Mw*rho*delP*1e3/Avo/gr - # rho and delP will be applied when reading the files in SEbudget function. - if key=='O3'+ext1_SE: - # for O3, we should not include fast cycling reactions - # As a result, we use below diagnostics in the model if available. If not, we use CHML and CHMP - if ((key+'_Loss' in ListVars) & (key+'_Prod' in ListVars)) : - dic_SE[var+'_CHML'][key+'_Loss'+ext1_SE]=MW[var]*1e3/AVO/gr - dic_SE[var+'_CHMP'][key+'_Prod'+ext1_SE]=MW[var]*1e3/AVO/gr - else: - if key+'_CHML' in ListVars: - dic_SE[var+'_CHML'][key+'_CHML'+ext1_SE]=MW[var]*1e3/AVO/gr - else: - dic_SE[var+'_CHML']['U'+ext1_SE]=0 - # End if - - if key+'_CHMP' in ListVars: - dic_SE[var+'_CHMP'][key+'_CHMP'+ext1_SE]=MW[var]*1e3/AVO/gr - else: - dic_SE[var+'_CHMP']['U'+ext1_SE]=0 - # End if - # End if - else: - if key+'_CHML' in ListVars: - dic_SE[var+'_CHML'][key+'_CHML'+ext1_SE]=MW[var]*1e3/AVO/gr - else: - dic_SE[var+'_CHML']['U'+ext1_SE]=0 - # End if - - if key+'_CHMP' in ListVars: - dic_SE[var+'_CHMP'][key+'_CHMP'+ext1_SE]=MW[var]*1e3/AVO/gr - else: - dic_SE[var+'_CHMP']['U'+ext1_SE]=0 - # End if - # End if - - - # for SF: - # original unit: [kg/m2/s] - if 'SF'+key in ListVars: - if var=='SO4': - dic_SE[var+'_SF']['SF'+key+ext1_SE]=32.066/115.11 - else: - dic_SE[var+'_SF']['SF'+key+ext1_SE]=1 - elif ((var=='DMS_OASISS') & ('OCN_FLUX_DMS' in ListVars)): - dic_SE[var+'_SF']['OCN_FLUX_DMS'+ext1_SE]=1 - - # End if - elif key+'SF' in ListVars: - dic_SE[var+'_SF'][key+ext1_SE+'SF']=1 - else: - dic_SE[var+'_SF']['PS'+ext1_SE]=0. - # End if - - - # for CLXF: - # original unit: [molec/cm2/s] - # conversion: Mw*10/Avo - if key+'_CLXF' in ListVars: - dic_SE[var+'_CLXF'][key+'_CLXF'+ext1_SE]=MW[var]*10/AVO # convert [molec/cm2/s] to [kg/m2/s] - else: - dic_SE[var+'_CLXF']['PS'+ext1_SE]=0. - # End if - - # Aerosols - if var in AEROSOLS: - # for each species: - # original unit : [kg/kg] in dry air - # convert to [kg/m2] - # conversion: delP/gr - # delP will be applied when reading the files in SEbudget function. - if key in ListVars: - if var=='SO4': # For SO4, we report all the budget calculation for Sulfur. - dic_SE[var+'_BURDEN'][key+ext1_SE]=(32.066/115.11)/gr - else: - dic_SE[var+'_BURDEN'][key+ext1_SE]=1/gr - # End if - else: - dic_SE[var+'_BURDEN']['U'+ext1_SE]=0 - # End if - - - # for DDF: - # original unit: [kg/m2/s] - if key+'DDF' in ListVars: - if var=='SO4': - dic_SE[var+'_DDF'][key+ext1_SE+'DDF']=32.066/115.11 - else: - dic_SE[var+'_DDF'][key+ext1_SE+'DDF']=1 - # End if - else: - dic_SE[var+'_DDF']['PS'+ext1_SE]=0. - # End if - - - # for SFWET: - # original unit: [kg/m2/s] - if key+'SFWET' in ListVars: - if var=='SO4': - dic_SE[var+'_WDF'][key+ext1_SE+'SFWET']=32.066/115.11 - else: - dic_SE[var+'_WDF'][key+ext1_SE+'SFWET']=1 - # End if - else: - dic_SE[var+'_WDF']['PS'+ext1_SE]=0. - # End if - - - # for sfgaex1: - # original unit: [kg/m2/s] - if key+'_sfgaex1' in ListVars: - if var=='SO4': - dic_SE[var+'_GAEX'][key+ext1_SE+'_sfgaex1']=32.066/115.11 - else: - dic_SE[var+'_GAEX'][key+ext1_SE+'_sfgaex1']=1 - # End if - else: - dic_SE[var+'_GAEX']['PS'+ext1_SE]=0. - # End if - - - # for DDF in cloud water: - # original unit: [kg/m2/s] - cloud_key=key[:-2]+'c'+key[-1] - if cloud_key+ext1_SE+'DDF' in ListVars: - if var=='SO4': - dic_SE[var+'_DDFC'][cloud_key+ext1_SE+'DDF']=32.066/115.11 - else: - dic_SE[var+'_DDFC'][cloud_key+ext1_SE+'DDF']=1 - # End if - else: - dic_SE[var+'_DDFC']['PS'+ext1_SE]=0. - # End if - - # for SFWET in cloud water: - # original unit: [kg/m2/s] - if cloud_key+ext1_SE+'SFWET' in ListVars: - if var=='SO4': - dic_SE[var+'_WDFC'][cloud_key+ext1_SE+'SFWET']=32.066/115.11 - else: - dic_SE[var+'_WDFC'][cloud_key+ext1_SE+'SFWET']=1 - # End if - else: - dic_SE[var+'_WDFC']['PS'+ext1_SE]=0. - # End if - - if var=='SO4': - # for Nucleation : - # original unit: [kg/m2/s] - if key+ext1_SE+'_sfnnuc1' in ListVars: - dic_SE[var+'_NUCL'][key+ext1_SE+'_sfnnuc1']=32.066/115.11 - else: - dic_SE[var+'_NUCL']['PS'+ext1_SE]=0. - # End if - - # for Aqueous phase : - # original unit: [kg/m2/s] - if (('AQSO4_H2O2'+ext1_SE in ListVars) & ('AQSO4_O3'+ext1_SE in ListVars)) : - dic_SE[var+'_AQS']['AQSO4_H2O2'+ext1_SE]=32.066/115.11 - dic_SE[var+'_AQS']['AQSO4_O3'+ext1_SE]=32.066/115.11 - else: - # original unit: [kg/m2/s] - if cloud_key+'AQSO4'+ext1_SE in ListVars: - dic_SE[var+'_AQS'][cloud_key+'AQSO4'+ext1_SE]=32.066/115.11 - else: - dic_SE[var+'_AQS']['PS'+ext1_SE]=0. - # End if - - if cloud_key+'AQH2SO4'+ext1_SE in ListVars: - dic_SE[var+'_AQS'][cloud_key+'AQH2SO4'+ext1_SE]=32.066/115.11 - else: - dic_SE[var+'_AQS']['PS'+ext1_SE]=0. - # End if - # End if - # End if - - else: # Gases - # for each species: - # original unit : [mole/mole] in dry air - # convert to [kg/m2] - # conversion: Mw*delP/Mwair/gr Mwair=28.97 gr/mole - # delP will be applied when reading the files in SEbudget function. - if key in ListVars: - dic_SE[var+'_BURDEN'][key+ext1_SE]=MW[var]/Mwair/gr - else: - dic_SE[var+'_BURDEN']['U'+ext1_SE]=0 - # End if - - # for DF: - # original unit: [kg/m2/s] - if 'DF_'+key in ListVars: - dic_SE[var+'_DDF']['DF_'+key+ext1_SE]=1 - else: - dic_SE[var+'_DDF']['PS'+ext1_SE]=0. - # End if - - # for WD: - # original unit: [kg/m2/s] - if 'WD_'+key in ListVars: - dic_SE[var+'_WDF']['WD_'+key+ext1_SE]=1 - else: - dic_SE[var+'_WDF']['PS'+ext1_SE]=0. - # End if - - # for Chem tendency: - # original unit: [kg/s] - # conversion: not needed - if 'D'+key+'CHM' in ListVars: - dic_SE[var+'_TEND']['D'+key+'CHM'+ext1_SE]=1 # convert [kg/s] to [kg/s] - else: - dic_SE[var+'_TEND']['U'+ext1_SE]=0 - # End if - - # for Lightning NO production: (always in gas) - # original unit: [Tg N/Yr] - # conversion: not needed - if 'LNO_COL_PROD' in ListVars: - dic_SE[var+'_LNO']['LNO_COL_PROD'+ext1_SE]=1 # convert [Tg N/yr] to [Tg N /yr] - else: - dic_SE[var+'_LNO']['PS'+ext1_SE]=0 - # End if - # End if (aerosols or gases) - # End for - # End for - - # Write to log - adfobj.debug_log(msg) - - return dic_SE -##### - - -def make_Dic_scn_var_comp(adfobj, variables, current_dir, dic_SE, current_files, ext1_SE, AEROSOLS,Tropospheric): - """ - This function retrieves the files, latitude, and longitude information - in all the directories within the chosen dates. - - current_dir: list - - showing the directories to look for files. always end with '/' - - current_files: list - - List of CAM history files - - start_year: string - - Starting year - - end_year: string - - Ending year - - kwargs - ------ - ext1_SE: string - - specify if the files are for only a region, which changes to variable names. - ex: if you saved files for a only a box region ($LL_lat$,$LL_lon$,$UR_lat$,$UR_lon$), - the 'lat' variable will be saved as: 'lat_$LL_lon$e_to_$UR_lon$e_$LL_lat$n_to_$UR_lat$n' - for instance: 'lat_65e_to_91e_20n_to_32n' - - Returns - ------- - Dic_crit: - - dictionary for critical values for current case - Dic_scn_var_comp: - - full dictionary of all variables and components for current case - - NOTE: The LNO is lightning NOx, which should be reported explicitly rather as CO_LNO, O3_LNO, ... - """ - - # Set lists to gather necessary variables for logging - missing_vars_tot = [] - needed_vars_tot = [] - - # Initialize final component dictionary - Dic_var_comp={} - - for current_var in variables: - if 'AOD' in current_var: - components=[current_var+'_AOD'] - else: - if current_var in AEROSOLS: # AEROSOLS - - # Components are: burden, chemical loss, chemical prod, dry deposition, - # surface emissions, elevated emissions, wet deposition, gas-aerosol exchange - components=[current_var+'_BURDEN',current_var+'_CHML',current_var+'_CHMP', - current_var+'_DDF',current_var+'_WDF', current_var+'_SF', current_var+'_CLXF', - current_var+'_DDFC',current_var+'_WDFC'] - - if current_var=='SO4': - # For SULF we also have AQS, nucleation, and strat-trop gas exchange - components.append(current_var+'_AQS') - components.append(current_var+'_NUCL') - components.append(current_var+'_GAEX') - components.remove(current_var+'_CHMP') - - #components.append(current_var+'_CLXF') # BRT - CLXF is added above. - if current_var == "SOA": - components.append(current_var+'_GAEX') - #End if - AEROSOLS - - else: # CHEMS - # Components are: burden, chemical loss, chemical prod, dry/wet deposition, - # surface emissions, elevated emissions, chemical tendency - # I always add Lightning NOx production when calculating O3 budget. - - components=[current_var+'_BURDEN',current_var+'_CHML',current_var+'_CHMP', - current_var+'_DDF',current_var+'_WDF', current_var+'_SF', current_var+'_CLXF', - current_var+'_TEND'] - - if current_var =="O3": - components.append(current_var+'_LNO') - # End if - # End if - msg = f"chem/aerosol tables: 'make_Dic_scn_var_comp'" - msg += f"\n\t Current CAM variable: {current_var}" - msg += f"\n\t Derived components for CAM variable {current_var}: {components}" - #adfobj.debug_log(msg) - Dic_comp={} - Dic_comp,missing_vars,needed_vars=SEbudget(adfobj,dic_SE,current_dir,current_files,components,ext1_SE) - - for comp in components: - # Write details to log file - msg += f"\n\t\t calculate derived component: {comp} for main variable, {current_var}" - adfobj.debug_log(msg) - - # Gather info for debugging - for var_m in missing_vars: - if var_m not in missing_vars_tot: - missing_vars_tot.append(var_m) - for var_n in needed_vars: - if var_n not in needed_vars_tot: - needed_vars_tot.append(var_n) - # End for - # End for - # Set dictionary for key of current variable with dictionary values of all - # necessary constituents for calculating the current variable - Dic_var_comp[current_var] = Dic_comp - Dic_scn_var_comp = Dic_var_comp - - # Critical threshholds, just run this once - # this is for finding tropospheric values - # Critical threshholds?\n", - # Just run this once\n", - tropospheric_method='NA' - try: - current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['O3'],ext1_SE) - Dic_crit=current_crit['O3'] - tropospheric_method='ozone' - msg += f"\n\t WARNING: Troposphere is defined as O3<150 ppb" - - except: - try: - current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['TROP_P','Pressure'],ext1_SE) - Dic_crit=current_crit #[['TROP_P','Pressure']] - tropospheric_method='tropopause' - msg += f"\n\t WARNING: Troposphere is defined as pressure>trop_p" - except: - current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['U'],ext1_SE) - Dic_crit=current_crit['U'] - Tropospheric=False - msg += f"\n\t WARNING: No way of defining troposphere was found in the model, budgets are total column" - # Log info to logging file - msg = f"chem/aerosol tables:" - msg += f"\n\t - potential missing variables from budget? {missing_vars_tot}" - adfobj.debug_log(msg) - - msg = f"chem/aerosol tables:" - msg += f"\n\t - needed variables for budget {needed_vars_tot}" - adfobj.debug_log(msg) - - return Dic_crit,Dic_scn_var_comp,Tropospheric,tropospheric_method -##### - - -def SEbudget(adfobj,dic_SE,data_dir,files,vars,ext1_SE,**kwargs): - """ - Function used for getting the data for the budget calculation. This is the - chunk of code that takes the longest by far. - - Example: - ** This is for both chemistry and aeorosl calculations - - dic_SE: dictionary specyfing what variables to get. For example, - for precipitation you can define SE as: - dic_SE['PRECT']={'PRECC'+ext1_SE:8.64e7,'PRECL'+ext1_SE:8.64e7} - - It means to sum the file variables "PRECC" and "PRECL" - for my arbitrary desired variable named "PRECT" - - - It also has the option to apply conversion factors. - For instance, PRECL and PRECC are in m/s. 8.64e7 is used to convernt m/s to mm/day - - - data_dir: string of the directory that contains the files. always end with '/' - - files: list of the files to be read - - var: string showing the variable to be extracted. - -> this will be the individual componnent, ie O3_CHMP, SOA_WDF, etc. - """ - - # gas constanct - Rgas=287.04 #[J/K/Kg]=8.314/0.028965 - - # Set lists to gather necessary variables for logging - missing_vars = [] - needed_vars = [] - Dic_all_data={} - -# all_data=[] - for file in range(len(files)): - - ds=xr.open_dataset(Path(data_dir) / files[file]) - - # Calculate these just once - if file==0: - mock_2d=np.zeros_like(np.array(ds['PS'+ext1_SE].isel(time=0))) - mock_3d=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) - - try: - delP=np.array(ds['PDELDRY'+ext1_SE].isel(time=0)) - except: - - hyai=np.array(ds['hyai']) - hybi=np.array(ds['hybi']) - - try: - PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) - except: - PS=np.array(ds['PS'+ext1_SE].isel(time=0)) - # End try/except - - P0=1e5 - Plevel=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) - - for i in range(len(Plevel)): - Plevel[i]=hyai[i]*P0+hybi[i]*PS - - delP=Plevel[1:]-Plevel[:-1] - - for var in vars: - if file == 0: - Dic_all_data[var]=[] - - - # Star gathering of variable data - - if var=='TROP_P': - data=np.array(ds['TROP_P'+ext1_SE].isel(time=0))/100 - elif var== 'Pressure': - try: - data=np.array(ds['PMID'+ext1_SE].isel(time=0))/100 - except: - hyam=np.array(ds['hyam']) - hybm=np.array(ds['hybm']) - try: - PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) - except: - PS=np.array(ds['PS'+ext1_SE].isel(time=0)) - P0=1e5 - data=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) - for i in range(len(data)): - data[i]=hyam[i]*P0+hybm[i]*PS - data=data/100 - else: - - - data=[] - for i in dic_SE[var].keys(): - - if file == 0: - msg = f"chem/aerosol tables: 'SEbudget'" - msg += f"\n\t\t ** variable(s) needed for derived var {var}: {dic_SE[var].keys()}" - msg += f"\n\t\t - constituent for derived var {var}: {i}" - adfobj.debug_log(msg) - if i not in needed_vars: - needed_vars.append(i) - if ((i!='PS'+ext1_SE) and (i!='U'+ext1_SE) ) : - data.append(np.array(ds[i].isel(time=0))*dic_SE[var][i]) - else: - if i=='PS'+ext1_SE: - data.append(mock_2d) - else: - data.append(mock_3d) - # End if - if file == 0: - - if var not in missing_vars: - if var!='U': # This is to avoid confusion between U variable or U mock! - missing_vars.append(var) - msg += f"\n\t\t - no variable was found for var {var}: {i}" - - # End if - - # Get total summed data for this history file data - data=np.sum(data,axis=0) - # End try/except - - if ('CHML' in var) or ('CHMP' in var) : - Temp=np.array(ds['T'+ext1_SE].isel(time=0)) - try: - Pres=np.array(ds['PMID'+ext1_SE].isel(time=0)) - except: - hyam=np.array(ds['hyam']) - hybm=np.array(ds['hybm']) - try: - PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) - except: - PS=np.array(ds['PS'+ext1_SE].isel(time=0)) - P0=1e5 - Pres=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) - for i in range(len(Pres)): - Pres[i]=hyam[i]*P0+hybm[i]*PS - rho= Pres/(Rgas*Temp) - data=data*delP/rho - elif ('BURDEN' in var): - data=data*delP - else: - data=data - # End if - # Add data to list - Dic_all_data[var].append(data) - ds.close() - for var in vars: # Take mean - Dic_all_data[var]=np.nanmean(Dic_all_data[var],axis=0) - - - return Dic_all_data,missing_vars,needed_vars -##### - - -def calc_budget_data(current_var, Dic_scn_var_comp, area, trop, inside, num_yrs, duration, AEROSOLS): - """ - Function to run through desired table values for calculations for the table entries - """ - - # Initialize full data dictionary for current table type - chem_dict = {} - - # Update variable marker if neccessary - if current_var == 'SO4': - specifier = ' S' - else: - specifier = '' - - # Calculate values for given variable - if 'AOD' in current_var: - # Burden - spc_burd = Dic_scn_var_comp[current_var][current_var+'_AOD'] - burden = np.ma.masked_where(inside==False,spc_burd) #convert Kg/m2 to Tg - BURDEN = np.ma.sum(burden*area)/np.ma.sum(area) - chem_dict[f"{current_var}_mean"] = np.round(BURDEN,5) - else: - # Surface Emissions - spc_sf = Dic_scn_var_comp[current_var][current_var+'_SF'] - tmp_sf = spc_sf - sf = np.ma.masked_where(inside==False,tmp_sf*area) #convert Kg/m2/s to Tg/yr - SF = np.ma.sum(sf*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_EMIS (Tg{specifier}/yr)"] = np.round(SF,5) - - # Elevated Emissions - spc_clxf = Dic_scn_var_comp[current_var][current_var+'_CLXF'] - tmp_clxf = spc_clxf - clxf = np.ma.masked_where(inside==False,tmp_clxf*area) #convert Kg/m2/s to Tg/yr - CLXF = np.ma.sum(clxf*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_EMIS_elevated (Tg{specifier}/yr)"] = np.round(CLXF,5) - - # Burden - spc_burd = Dic_scn_var_comp[current_var][current_var+'_BURDEN'] - spc_burd = np.where(np.isnan(trop),np.nan,spc_burd) - tmp_burden = np.nansum(spc_burd*area,axis=0) - burden = np.ma.masked_where(inside==False,tmp_burden) #convert Kg/m2 to Tg - BURDEN = np.ma.sum(burden*1e-9) - chem_dict[f"{current_var}_BURDEN (Tg{specifier})"] = np.round(BURDEN,5) - - # Chemical Loss - spc_chml = Dic_scn_var_comp[current_var][current_var+'_CHML'] - spc_chml = np.where(np.isnan(trop),np.nan,spc_chml) - tmp_chml = np.nansum(spc_chml*area,axis=0) - chml = np.ma.masked_where(inside==False,tmp_chml) #convert Kg/m2/s to Tg/yr - CHML = np.ma.sum(chml*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_CHEM_LOSS (Tg{specifier}/yr)"] = np.round(CHML,5) - - # Chemical Production - if current_var == 'SO4': # chemical production is basically the elevated emissions. - # We have removed it for SO4 budget. and put 0 here, so, we don't report it - chem_dict[f"{current_var}_CHEM_PROD (Tg{specifier}/yr)"] = 0 - else: - spc_chmp = Dic_scn_var_comp[current_var][current_var+'_CHMP'] - spc_chmp = np.where(np.isnan(trop),np.nan,spc_chmp) - tmp_chmp = np.nansum(spc_chmp*area,axis=0) - chmp = np.ma.masked_where(inside==False,tmp_chmp) #convert Kg/m2/s to Tg/yr - CHMP = np.ma.sum(chmp*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_CHEM_PROD (Tg{specifier}/yr)"] = np.round(CHMP,5) - # End if - - # Aerosol calculations - #--------------------- - if current_var in AEROSOLS: - - # Dry Deposition Flux - spc_ddfa = Dic_scn_var_comp[current_var][current_var+'_DDF'] - spc_ddfc = Dic_scn_var_comp[current_var][current_var+'_DDFC'] - spc_ddf = spc_ddfa +spc_ddfc - tmp_ddf = spc_ddf - ddf = np.ma.masked_where(inside==False,tmp_ddf*area) #convert Kg/m2/s to Tg/yr - DDF = np.ma.sum(ddf*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_DRYDEP (Tg{specifier}/yr)"] = np.round(DDF,5) - - # Wet deposition - spc_wdfa = Dic_scn_var_comp[current_var][current_var+'_WDF'] - spc_wdfc = Dic_scn_var_comp[current_var][current_var+'_WDFC'] - spc_wdf = spc_wdfa +spc_wdfc - tmp_wdf = spc_wdf - wdf = np.ma.masked_where(inside==False,tmp_wdf*area) #convert Kg/m2/s to Tg/yr - WDF = -1*np.ma.sum(wdf*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_WETDEP (Tg{specifier}/yr)"] = np.round(WDF,5) - - if current_var in ["SOA",'SO4']: - # gas-aerosol Exchange - spc_gaex = Dic_scn_var_comp[current_var][current_var+'_GAEX'] - tmp_gaex = spc_gaex - gaex = np.ma.masked_where(inside==False,tmp_gaex*area) #convert Kg/m2/s to Tg/yr - GAEX = np.ma.sum(gaex*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_GAEX (Tg{specifier}/yr)"] = np.round(GAEX,5) - - # LifeTime = Burden/(loss+deposition) no chemical loss for aerosols - LT = BURDEN/(DDF+WDF)* duration/86400/num_yrs # days - chem_dict[f"{current_var}_LIFETIME (days)"] = np.round(LT,5) - - if current_var == 'SO4': - # Aqueous Chemistry - spc_aqs = Dic_scn_var_comp[current_var][current_var+'_AQS'] - tmp_aqs = spc_aqs - aqs = np.ma.masked_where(inside==False,tmp_aqs*area) #convert Kg/m2/s to Tg/yr - AQS = np.ma.sum(aqs*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_AQUEOUS (Tg{specifier}/yr)"] = np.round(AQS,5) - - # Nucleation - spc_nucl = Dic_scn_var_comp[current_var][current_var+'_NUCL'] - tmp_nucl = spc_nucl - nucl = np.ma.masked_where(inside==False,tmp_nucl*area) #convert Kg/m2/s to Tg/yr - NUCL = np.ma.sum(nucl*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_NUCLEATION (Tg{specifier}/yr)"] = np.round(NUCL,5) - - # Gaseous calculations - #--------------------- - else: - # Dry Deposition Flux - spc_ddf = Dic_scn_var_comp[current_var][current_var+'_DDF'] - tmp_ddf = spc_ddf - ddf = np.ma.masked_where(inside==False,tmp_ddf*area) #convert Kg/m2/s to Tg/yr - DDF = np.ma.sum(ddf*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_DRYDEP (Tg/yr)"] = np.round(DDF,5) - - # Wet Deposition Flux - spc_wdf = Dic_scn_var_comp[current_var][current_var+'_WDF'] - tmp_wdf = spc_wdf - wdf = np.ma.masked_where(inside==False,tmp_wdf*area) #convert Kg/m2/s to Tg/yr - WDF = -1*np.ma.sum(wdf*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_WETDEP (Tg/yr)"] = np.round(WDF,5) - - # Total Deposition - TDEP = DDF + WDF - chem_dict[f"{current_var}_TDEP (Tg/yr)"] = np.round(TDEP,5) - - # LifeTime = Burden/(loss+deposition) - if current_var == "CH4": - LT = BURDEN/(CHML+DDF+WDF) # years - chem_dict[f"{current_var}_LIFETIME (years)"] = np.round(LT,5) - else: - if (CHML+DDF+WDF) > 0: - if CHML != 0: - LT = BURDEN/(CHML+DDF+WDF)*duration/86400/num_yrs # days - chem_dict[f"{current_var}_LIFETIME (days)"] = np.round(LT,5) - else: - # do not report lifetime if chemical loss (for gases) is not included in the model outputs - # and put 0 here, so, we don't report it - chem_dict[f"{current_var}_LIFETIME (days)"] = 0 - # End if - # End if - # End if - - #NET = CHMP-CHML - # Chemical Tendency - spc_tnd = Dic_scn_var_comp[current_var][current_var+'_TEND'] - spc_tnd = np.where(np.isnan(trop),np.nan,spc_tnd) - tmp_tnd = np.nansum(spc_tnd,axis=0) - tnd = np.ma.masked_where(inside==False,tmp_tnd) #convert Kg/s to Tg/yr - TND = np.ma.sum(tnd*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_TEND (Tg/yr)"] = np.round(TND,5) - - # O3 dependent calculations - if current_var == "O3": - # Stratospheric-Tropospheric Exchange - STE = DDF-TND - chem_dict[f"{current_var}_STE (Tg/yr)"] = np.round(STE,5) - - # Lightning NOX production - spc_lno = Dic_scn_var_comp[current_var][current_var+'_LNO'] - tmp_lno = np.ma.masked_where(inside==False,spc_lno) - LNO = np.ma.sum(tmp_lno) - chem_dict[f"{current_var}_LNO (Tg N/yr)"] = np.round(LNO,5) - # End if (aerosol or gas) - return chem_dict -##### - - -def make_table(adfobj, vars, chem_type, Dic_scn_var_comp, areas, trops, case_names, nicknames, durations, insides, num_yrs, AEROSOLS): - """ - Create CSV table for aeorosols and gases, if applicable - - Table includes column values of variable, case(s), difference (if applicable) - - If this is a single model vs model run: 4 columns - first column: variables names, - second column: test case variable values - third column: baseline case variable values - final column: difference of test and baseline. - If this is a model vs obs run: 2 columns - first column: variables names, - second column: test case variable values - """ - # Initialize an empty dictionary to store DataFrames - dfs = {} - - #Special ADF variable which contains the output paths for - #all generated plots and tables for each case: - output_locs = adfobj.plot_location - - #Convert output location string to a Path object: - output_location = Path(output_locs[0]) - - # Loop over model cases - - for i,case in enumerate(nicknames): - - nickname = case - - # Collect row data in a list of dictionaries - #durations[case] - rows = [] - for current_var in vars: - chem_dict = calc_budget_data(current_var, Dic_scn_var_comp[case], areas[case], trops[case], insides[case], - num_yrs[case], durations[case], AEROSOLS) - # Loop through table variables - for key, val in chem_dict.items(): - if val != 0: # Skip variables with a value of 0 - print(f"\t - Variable '{key}' being added to table") - rows.append({'variable': key, nickname: np.round(val, 3)}) - elif 'OASISS_EMIS (' in key: # the paranthesis is to ignore EMIS_Elevated variables! - print(f"\t - Variable '{key}' being added to table") - rows.append({'variable': key, nickname: np.round(val, 3)}) - else: - msg = f"chem/aerosol tables:" - msg += f"\n\t - Variable '{key}' has value of 0, will not add to table" - adfobj.debug_log(msg) - # End if - # End for - # End for - - # Create the DataFrame for the current case - table_df = pd.DataFrame(rows) - - if chem_type == 'gases': - # Replace compound names directly in the DataFrame - replacements = { - 'MTERP': 'Monoterpene', - 'CH3OH': 'Methanol', - 'CH3COCH3': 'Acetone', - 'O3_LNO': 'LNOx_PROD' - } - table_df['variable'] = table_df['variable'].replace(replacements, regex=True) - # End if - - # Store the DataFrame in the dictionary - dfs[nickname] = table_df - - # End for - - # Merge the DataFrames on the 'variable' column - if len(case_names) == 2: - - table_df = pd.merge(dfs[nicknames[0]], dfs[nicknames[1]], on='variable') - - # Calculate the differences between case columns - table_df['difference'] = table_df[nicknames[0]] - table_df[nicknames[1]] - - #Create output file name: - output_csv_file = output_location / f'ADF_amwg_{chem_type}_table.csv' - # Save table to CSV and add table dataframe to website (if enabled) - table_df.to_csv(output_csv_file, index=False) - #adfobj.add_website_data(table_df, chem_type, case, plot_type="Tables") - adfobj.add_website_data(table_df, chem_type, case_names[0], plot_type="Tables") - -##### diff --git a/scripts/analysis/aerosol_gas_tables_Tropopause_version1.py b/scripts/analysis/aerosol_gas_tables_Tropopause_version1.py deleted file mode 100644 index 874f884b7..000000000 --- a/scripts/analysis/aerosol_gas_tables_Tropopause_version1.py +++ /dev/null @@ -1,1379 +0,0 @@ -import numpy as np -import xarray as xr -import sys -from pathlib import Path -import warnings # use to warn user about missing files. - -from datetime import datetime -import numpy as np -import itertools - -try: - import pandas as pd -except ImportError: - print("Pandas module does not exist in python path, but is needed for amwg_table.") - print("Please install module, e.g. 'pip install pandas'.") - sys.exit(1) -#End except - -# Import necessary ADF modules: -from adf_base import AdfError - -def aerosol_gas_tables(adfobj): - ''' - Calculate aerosol and gaseous budget tables - - Default set of variables: change in lib/adf_variable_defaults.yaml - ------------------------- - GAS_VARIABLES: ['CH4','CH3CCL3', 'CO', 'O3', 'ISOP', 'MTERP', 'CH3OH', 'CH3COCH3'] - AEROSOL_VARIABLES: ['AOD','SOA', 'SALT', 'DUST', 'POM', 'BC', 'SO4'] - - Default output for tables: - - Gases: - ------ - CH4_BURDEN (Tg), CH4_CHEM_LOSS (Tg/yr), CH4_LIFETIME (years) - - CH3CCL3_BURDEN (Tg), CH3CCL3_CHEM_LOSS (Tg/yr), CH3CCL3_LIFETIME (days) - - CO_EMIS (Tg/yr), CO_BURDEN (Tg), CO_CHEM_LOSS (Tg/yr), CO_CHEM_PROD (Tg/yr), CO_DRYDEP (Tg/yr) - CO_TDEP (Tg/yr), CO_LIFETIME (days), CO_TEND (Tg/yr) - - O3_BURDEN (Tg), O3_CHEM_LOSS (Tg/yr), O3_CHEM_PROD (Tg/yr), O3_DRYDEP (Tg/yr), O3_TDEP (Tg/yr) - O3_LIFETIME (days), O3_TEND (Tg/yr), O3_STE (Tg/yr) - - LNOx_PROD (Tg N/yr) - - ISOP_EMIS (Tg/yr), ISOP_BURDEN (Tg) - - Monoterpene_EMIS (Tg/yr), Monoterpene_BURDEN (Tg) - - Methanol_EMIS (Tg/yr), Methanol_BURDEN (Tg), Methanol_DRYDEP (Tg/yr), Methanol_WETDEP (Tg/yr), Methanol_TDEP (Tg/yr) - - Acetone_EMIS (Tg/yr), Acetone_BURDEN (Tg), Acetone_DRYDEP (Tg/yr), Acetone_WETDEP (Tg/yr), Acetone_TDEP (Tg/yr) - - - - Aerosols: - --------- - AOD_mean - - SOA_BURDEN (Tg), SOA_CHEM_LOSS (Tg/yr), SOA_DRYDEP (Tg/yr), SOA_WETDEP (Tg/yr), SOA_GAEX (Tg/yr), SOA_LIFETIME (days) - - SALT_EMIS (Tg/yr), SALT_BURDEN (Tg), SALT_DRYDEP (Tg/yr), SALT_WETDEP (Tg/yr), SALT_LIFETIME (days) - - DUST_EMIS (Tg/yr), DUST_BURDEN (Tg), DUST_DRYDEP (Tg/yr), DUST_WETDEP (Tg/yr), DUST_LIFETIME (days) - - POM_EMIS (Tg/yr), POM_BURDEN (Tg), POM_DRYDEP (Tg/yr), POM_WETDEP (Tg/yr), POM_LIFETIME (days) - - BC_EMIS (Tg/yr), BC_BURDEN (Tg), BC_DRYDEP (Tg/yr), BC_WETDEP (Tg/yr), BC_LIFETIME (days) - - SO4_EMIS_elevated (Tg S/yr), SO4_BURDEN (Tg S), SO4_DRYDEP (Tg S/yr), SO4_WETDEP (Tg S/yr), SO4_GAEX (Tg S/yr) - SO4_LIFETIME (days), SO4_AQUEOUS (Tg S/yr), SO4_NUCLEATION (Tg S/yr) - - - List of variable names and descriptions for clarity - --------------------------------------------------- - - ListVars: list of all available variables from given history file - - GAS_VARIABLES: list fo necessary CAM gaseous variables - - AEROSOL_VARIABLES: list fo necessary CAM aerosol variables - - AEROSOLS: list of necessary aerosols for computations - - - MODIFICATION HISTORY: - Behrooz Roozitalab, 02, NOV, 2022: VERSION 1.00 - - Initial version - - Justin Richling, 27 Nov, 2023 - - updated to fit to ADF and check with old AMWG chem/aerosol tables - - fixed: - * added difference bewtween cases column to tables - - Behrooz Roozitalab, 8 Aug, 2024 - - fixed: - * lifetime inconsitencies - * Removed redundant calculations to improve the speed - * Verified the results against the NCL script. - - Behrooz Roozitalab, 5 Jun, 2025 - - fixed: - * Fix the bugs in the calculation (when converting from Jupyterhub to ADF) - * add the 'U' variable in dic_SE - * make the code faster by modifying make_Dic_scn_var_comp - * Add a condition to calculate whole world budgets when O3 is not find. - * Update pressure calculation in a more general way. - - Behrooz Roozitalab, 20 Aug, 2025 _ Version 1 - - fixed: - * the html page was not created, it is fixed. - * added "hm" as a case to enable using annual averaged files in addition to monthly files. - * This version uses 500hPa as the tropopause threshold. Use Version 0 for a realistic case. - * Added DMS to gases list - reported as DMS not S - * Automatic addition of gaseous compounds even when not defined in the default list, - * based on Carbon MW (12). It still needs ADF modification to read a list from yaml file. - ''' - - - #Notify user that script has started: - msg = "\n Calculating chemistry/aerosol budget tables..." - print(f"{msg}\n {'-' * (len(msg)-3)}") - - # Inputs - #------- - # Variable defaults info - res = adfobj.variable_defaults # dict of variable-specific plot preferences - bres = res['budget_tables'] - # list of the gaseous variables to be caculated. - GAS_VARIABLES = bres['GAS_VARIABLES'] - - # list of the aerosol variables to be caculated. - AEROSOL_VARIABLES = bres['AEROSOL_VARIABLES'] - - #list of all the variables to be caculated. - VARIABLES = GAS_VARIABLES + AEROSOL_VARIABLES - - # For the case that outputs are saved for a specific region. - # i.e., when using fincllonlat in user_nl_cam - ext1_SE = bres['ext1_SE'] - - # Tropospheric Values - # ------------------- - # if True, calculate only Tropospheric values - # if False, all layers - # tropopause is defiend as either directly or indirectly. Look for tropopause to see the definition - Tropospheric = bres['Tropospheric'] - - ### NOT WORKING FOR NOW - # To calculate the budgets only for a region - # Lat/Lon extent - limit = bres['limit'] - regional = bres['regional'] - - # Dictionary for Molecular weights. Keys must be consistent with variable name - # For aerosols, the MW is used only for chemical loss, chemical production, and elevated emission calculations - # For SO4, we report everything in terms of Sulfur, so we use Sulfur MW here - MW = bres['MW'] - - # automatic generation of MW - for var in VARIABLES: - if var not in MW.keys(): - print(f"using Carbon molecular weight for {var}") - MW[var]=12 - - - # Avogadro's Number - AVO = float(bres['AVO']) - # gravity - gr = float(bres['gr']) - # Mw air - Mwair = float(bres['Mwair']) - - # The variables in the list below must be aerosols - do not add AOD and DAOD - # no need to change this list, unless for a specific need! - AEROSOLS = bres['AEROSOLS'] - - # Start gathering case, path, and data info - #----------------------------------------- - - # CAM simulation variables (these quantities are always lists): - case_names = adfobj.get_cam_info('cam_case_name', required=True) - - # Grab all case nickname(s) - test_nicknames_list = adfobj.case_nicknames["test_nicknames"] - nicknames_list = test_nicknames_list - # Grab climo years - start_years = adfobj.climo_yrs["syears"] - end_years = adfobj.climo_yrs["eyears"] - - #Grab history strings: - hist_strs = adfobj.hist_string["test_hist_str"] - - # Grab history file locations from config yaml file - hist_locs = adfobj.get_cam_info("cam_hist_loc", required=True) - - # Check if this is test model vs baseline model - # If so, update test case(s) lists created above - if not adfobj.compare_obs: - # Get baseline case info - case_names += [adfobj.get_baseline_info("cam_case_name")] - nicknames_list += [adfobj.case_nicknames["base_nickname"]] - - # Grab climo years - start_years += [adfobj.climo_yrs["syear_baseline"]] - end_years += [adfobj.climo_yrs["eyear_baseline"]] - - # Get history file info - hist_strs += [adfobj.hist_string["base_hist_str"]] - hist_locs += [adfobj.get_baseline_info("cam_hist_loc")] - # End if - - # Check to ensure number of case names matches number history file locations. - # If not, exit script - if len(hist_locs) != len(case_names): - errmsg = "Error: number of cases does not match number of history file locations. Script is exiting." - raise AdfError(errmsg) - - # Initialize nicknames dictionary - #nicknames = {} - - # Filter the list to include only strings that are possible h0 strings - # - Search for either h0 or h0a - substrings = {"cam.h0","cam.h0a","cam.hm"} - case_hist_strs = [] - for cam_case_str in hist_strs: - # Check each possible h0 string - for string in cam_case_str: - if string in substrings: - case_hist_strs.append(string) - break - - # Create path object for the CAM history file(s) location: - data_dirs = [] - for case_idx,case in enumerate(nicknames_list): - - print(f"\t Looking for history location: {hist_locs[case_idx]}") - - - #Check that history file input directory actually exists: - if (hist_locs[case_idx] is None) or (not Path(hist_locs[case_idx]).is_dir()): - errmsg = f"History files directory '{hist_locs[case_idx]}' not found. Script is exiting." - raise AdfError(errmsg) - - #Write to debug log if enabled: - adfobj.debug_log(f"DEBUG: location of history files is {str(hist_locs[case_idx])}") - # Update list for found directories - data_dirs.append(hist_locs[case_idx]) - - # End gathering case, path, and data info - #----------------------------------------- - # Periods of Interest - # ------------------- - # choose the period of interest. Plots will be averaged within this period - durations = {} - num_yrs = {} - - # Main function - #-------------- - # Set dictionary of components for each case - Dic_scn_var_comp = {} - areas = {} - trops = {} - insides = {} - for i,case in enumerate(nicknames_list): - - start_year = start_years[i] - end_year = end_years[i] + 1 - start_date = f"{start_year}-1-1" - end_date = f"{end_year}-1-1" - - # Create time periods - start_period = datetime.strptime(start_date, "%Y-%m-%d") - end_period = datetime.strptime(end_date, "%Y-%m-%d") - - # Calculated duration of time period in seconds? - durations[case] = (end_period-start_period).days*86400 #+365*86400 - - - # Get number of years for calculations - num_yrs[case] = (int(end_year)-int(start_year)) #+1 - - # Get currenty history file directory - data_dir = data_dirs[i] - - # Get all files, lats, lons, and area weights for current case - Files,Lats,Lons,areas[case],ext1_SE = Get_files(adfobj,data_dir,start_year,end_year,case_hist_strs[i],area=True) - # find the name of all the variables in the file. - # this will help the code to work for the variables that are not in the files (assingn 0s) - tmp_file = xr.open_dataset(Path(data_dir) / Files[0]) - ListVars = list(tmp_file.variables) - - # Set up and fill dictionaries for components for current cases - dic_SE = set_dic_SE(ListVars,ext1_SE,VARIABLES) - dic_SE = fill_dic_SE(adfobj, dic_SE, VARIABLES, ListVars, ext1_SE, AEROSOLS, MW, AVO, gr, Mwair) - - text = f'\n\t Calculating values for {case}' - print(text) - print("\t " + "-" * (len(text) - 2)) - - # Gather dictionary data for current case - # NOTE: The calculations can take a long time... - Dic_crit, Dic_scn_var_comp[case],Tropospheric,tropospheric_method = make_Dic_scn_var_comp(adfobj, VARIABLES, data_dir, dic_SE, Files, ext1_SE, AEROSOLS,Tropospheric) - # Regional refinement - # NOTE: This function 'Inside_SE' is unavailable at the moment! - JR 10/2024 - if regional: - #inside = Inside_SE_region(current_lat,current_lon,dir_shapefile) - inside = Inside_SE(Lats,Lons,limit) - else: - if len(np.shape(areas[case])) == 1: - inside = np.full((len(Lons)),True) - else: - inside = np.full((len(Lats),len(Lons)),True) - - # Set critical threshold - current_crit = Dic_crit - if Tropospheric: - if tropospheric_method=='pressure': - # using pressure > 500hPa - trop = np.where(current_crit<500,np.nan,current_crit) - elif tropospheric_method=='NA': - print('ERROR: Tropopause is not defined correctly!') - else: - trop=current_crit - trops[case] = trop - insides[case] = inside - - # Make and save tables - table_kwargs = {"adfobj":adfobj, - "Dic_scn_var_comp":Dic_scn_var_comp, - "areas":areas, - "trops":trops, - "case_names":case_names, - "nicknames":nicknames_list, - "durations":durations, - "insides":insides, - "num_yrs":num_yrs, - "AEROSOLS":AEROSOLS} - - #print(table_kwargs) - - # Create the budget tables - #------------------------- - # Aerosols - if len(AEROSOL_VARIABLES) > 0: - print("\tMaking table for aerosols") - make_table(vars=AEROSOL_VARIABLES, chem_type='aerosols', **table_kwargs) - # Gases - if len(GAS_VARIABLES) > 0: - print("\tMaking table for gases") - make_table(vars=GAS_VARIABLES, chem_type='gases', **table_kwargs) -####### - -################## -# Helper functions -################## - -def list_files(adfobj, directory, start_year ,end_year, h_case): - - """ - This function extracts the files in the directory that are within the chosen dates - and history number. - """ - - # History file year range - yrs = np.arange(int(start_year), int(end_year)) - - all_filenames = [] - for i in yrs: -# all_filenames.append(sorted(Path(directory).glob(f'*.{h_case}.{i}-*'))) - all_filenames.append(sorted(Path(directory).glob(f'*.{h_case}.{i}*'))) - - #print(directory) - # Flattening the list of lists - filenames = list(itertools.chain.from_iterable(sorted(all_filenames))) - if len(filenames)==0: - #sys.exit(" Directory has no outputs ") - msg = f"chem/aerosol tables, 'list_files':" - msg += f"\n\t - Directory '{directory}' has no outputs." - adfobj.debug_log(msg) - - return filenames -##### - - -def Get_files(adfobj, data_dir, start_year, end_year, h_case, **kwargs): - - """ - This function retrieves the files, latitude, and longitude information - in all the directories within the chosen dates. - """ - ext1_SE = kwargs.pop('ext1_SE','') - area = kwargs.pop('area',False) - - Earth_rad=6.371e6 # Earth Radius in meters - - current_files = list_files(adfobj, data_dir, start_year, end_year,h_case) - # get the Lat and Lons for each case - tmp_file = xr.open_dataset(Path(data_dir) / current_files[0]) - lon = tmp_file['lon'+ext1_SE].data - lon[lon > 180.] -= 360 # shift longitude from 0-360˚ to -180-180˚ - lat = tmp_file['lat'+ext1_SE].data - - if area == True: - try: - tmp_area = tmp_file['area'+ext1_SE].data - Earth_area = 4 * np.pi * Earth_rad**(2) - - areas = tmp_area*Earth_area/np.nansum(tmp_area) - except KeyError: - try: - tmp_area = tmp_file['AREA'+ext1_SE].isel(time=0).data - areas=tmp_area - #Earth_area = 4 * np.pi * Earth_rad**(2) - #areas = tmp_area*Earth_area/np.nansum(tmp_area) - except: - dlon = np.abs(lon[1]-lon[0]) - dlat = np.abs(lat[1]-lat[0]) - - lon2d,lat2d = np.meshgrid(lon,lat) - #area=np.zeros_like(lat2d) - - dy = Earth_rad*dlat*np.pi/180 - dx = Earth_rad*np.cos(lat2d*np.pi/180)*dlon*np.pi/180 - - tmp_area = dx*dy - areas = tmp_area - # End if - - # Variables to return - return current_files,lat,lon,areas,ext1_SE -##### - -def set_dic_SE(ListVars, ext1_SE,variables): - """ - Initialize dictionary to house all the relevant tabel data - """ - - # Initialize dictionary - #---------------------- - dic_SE={} - - # Chemistry - #---------- - dic_SE['U']={'U'+ext1_SE:1} - dic_SE['O3']={'O3'+ext1_SE:1e9} # covert to ppb for Tropopause calculation - dic_SE['CH4']={'CH4'+ext1_SE:1} - dic_SE['CO']={'CO'+ext1_SE:1} - - dic_SE['ISOP']={'ISOP'+ext1_SE:1} - dic_SE['MTERP']={'MTERP'+ext1_SE:1} - dic_SE['CH3OH']={'CH3OH'+ext1_SE:1} - dic_SE['CH3COCH3']={'CH3COCH3'+ext1_SE:1} - dic_SE['CH3CCL3']={'CH3CCL3'+ext1_SE:1} - dic_SE['CHBR3']={'CHBR3'+ext1_SE:1} - dic_SE['CH2BR2']={'CH2BR2'+ext1_SE:1} - - # Aerosols - #--------- - - dic_SE['DAOD']={'AODDUSTdn'+ext1_SE:1} - dic_SE['AOD']={'AODVISdn'+ext1_SE:1} - - dic_SE['DUST']={'dst_a1'+ext1_SE:1, - 'dst_a2'+ext1_SE:1, - 'dst_a3'+ext1_SE:1} - - dic_SE['SALT']={'ncl_a1'+ext1_SE:1, - 'ncl_a2'+ext1_SE:1, - 'ncl_a3'+ext1_SE:1} - - dic_SE['POM']={'pom_a1'+ext1_SE:1, - 'pom_a4'+ext1_SE:1} - - dic_SE['BC']={'bc_a1'+ext1_SE:1, - 'bc_a4'+ext1_SE:1} - - - dic_SE['SO4']={'so4_a1'+ext1_SE:1, - 'so4_a2'+ext1_SE:1, - 'so4_a3'+ext1_SE:1, - 'so4_a5'+ext1_SE:1} - - # FOR SOA, first check if the integrated bins are included - if (('soa_a1'+ext1_SE in ListVars ) & ('soa_a1'+ext1_SE in ListVars )): - dic_SE['SOA'] = {'soa_a1'+ext1_SE:1, - 'soa_a2'+ext1_SE:1} - else: - dic_SE['SOA'] = {'soa1_a1'+ext1_SE:1, - 'soa2_a1'+ext1_SE:1, - 'soa3_a1'+ext1_SE:1, - 'soa4_a1'+ext1_SE:1, - 'soa5_a1'+ext1_SE:1, - 'soa1_a2'+ext1_SE:1, - 'soa2_a2'+ext1_SE:1, - 'soa3_a2'+ext1_SE:1, - 'soa4_a2'+ext1_SE:1, - 'soa5_a2'+ext1_SE:1} - - dic_SE['DMS']={'DMS'+ext1_SE:1} - #dic_SE['TROP_P']={'TROP_P'+ext1_SE:1} - - - # automatic generation of dic_SE - for var in variables: - if var not in dic_SE.keys(): - dic_SE[var]={var+ext1_SE:1} - - # consider for OASISS DMS separately - if var=='DMS': - dic_SE['DMS_OASISS']={'DMS_OASISS'+ext1_SE:1} - # End if - - return dic_SE -##### - -def fill_dic_SE(adfobj, dic_SE, variables, ListVars, ext1_SE, AEROSOLS, MW, AVO, gr, Mwair): - """ - Function for dealing with conversion factors for different components and filling the main data - dictionary 'dic_SE' - - Input dictionary and return updated dictionary 'dic_SE' - - Arguments - --------- - variables : list - - list of main variables? - ListVars : list - - list of ??????? - - Returns - ------- - dic_SE : dict - - full dictionary of derived variables - - Some conversion factors need density or Layer's pressure, that will be accounted for when reading the files. - We convert everying to kg/m2/s or kg/m2 or kg/s, so that final Tg/yr or Tg results are consistent - """ - - # Logging info message - msg = f"chem/aerosol tables: 'fill_dic_SE'" - - for var in variables: - - if 'AOD' in var: - dic_SE[var+'_AOD']={} - else: - dic_SE[var+'_BURDEN']={} - dic_SE[var+'_CHML']={} - dic_SE[var+'_CHMP']={} - - dic_SE[var+'_SF']={} - dic_SE[var+'_CLXF']={} - - dic_SE[var+'_DDF']={} - dic_SE[var+'_WDF']={} - - if var in AEROSOLS: - dic_SE[var+'_GAEX']={} - dic_SE[var+'_DDFC']={} - dic_SE[var+'_WDFC']={} - else: - dic_SE[var+'_TEND']={} - dic_SE[var+'_LNO']={} - # End if - - # We have nucleation and aqueous chemistry for sulfate. - if var=='SO4': - dic_SE[var+'_NUCL']={} - dic_SE[var+'_AQS']={} - # End if - - # Grab the variable keys - var_keys = dic_SE[var].keys() - - for key in var_keys: - msg += f"\n\t Creating component of {var}: {key}" - - # for CHML and CHMP: - # original unit : [molec/cm3/s] - # following Tilmes code to convert to [kg/m2/s] - # conversion: Mw*rho*delP*1e3/Avo/gr - # rho and delP will be applied when reading the files in SEbudget function. - - # for AOD and DAOD: - if 'AOD' in var: - if key in ListVars: - dic_SE[var+'_AOD'][key+ext1_SE]=1 - else: - dic_SE[var+'_AOD']['PS'+ext1_SE]=0. - # End if - continue # AOD doesn't need any other budget calculations - # End if - - # for CHML and CHMP: - # original unit : [molec/cm3/s] - # following Tilmes code to convert to [kg/m2/s] - # conversion: Mw*rho*delP*1e3/Avo/gr - # rho and delP will be applied when reading the files in SEbudget function. - if key=='O3'+ext1_SE: - # for O3, we should not include fast cycling reactions - # As a result, we use below diagnostics in the model if available. If not, we use CHML and CHMP - if ((key+'_Loss' in ListVars) & (key+'_Prod' in ListVars)) : - dic_SE[var+'_CHML'][key+'_Loss'+ext1_SE]=MW[var]*1e3/AVO/gr - dic_SE[var+'_CHMP'][key+'_Prod'+ext1_SE]=MW[var]*1e3/AVO/gr - else: - if key+'_CHML' in ListVars: - dic_SE[var+'_CHML'][key+'_CHML'+ext1_SE]=MW[var]*1e3/AVO/gr - else: - dic_SE[var+'_CHML']['U'+ext1_SE]=0 - # End if - - if key+'_CHMP' in ListVars: - dic_SE[var+'_CHMP'][key+'_CHMP'+ext1_SE]=MW[var]*1e3/AVO/gr - else: - dic_SE[var+'_CHMP']['U'+ext1_SE]=0 - # End if - # End if - else: - if key+'_CHML' in ListVars: - dic_SE[var+'_CHML'][key+'_CHML'+ext1_SE]=MW[var]*1e3/AVO/gr - else: - dic_SE[var+'_CHML']['U'+ext1_SE]=0 - # End if - - if key+'_CHMP' in ListVars: - dic_SE[var+'_CHMP'][key+'_CHMP'+ext1_SE]=MW[var]*1e3/AVO/gr - else: - dic_SE[var+'_CHMP']['U'+ext1_SE]=0 - # End if - # End if - - - # for SF: - # original unit: [kg/m2/s] - if 'SF'+key in ListVars: - if var=='SO4': - dic_SE[var+'_SF']['SF'+key+ext1_SE]=32.066/115.11 - else: - dic_SE[var+'_SF']['SF'+key+ext1_SE]=1 - elif ((var=='DMS_OASISS') & ('OCN_FLUX_DMS' in ListVars)): - dic_SE[var+'_SF']['OCN_FLUX_DMS'+ext1_SE]=1 - - # End if - elif key+'SF' in ListVars: - dic_SE[var+'_SF'][key+ext1_SE+'SF']=1 - else: - dic_SE[var+'_SF']['PS'+ext1_SE]=0. - # End if - - - # for CLXF: - # original unit: [molec/cm2/s] - # conversion: Mw*10/Avo - if key+'_CLXF' in ListVars: - dic_SE[var+'_CLXF'][key+'_CLXF'+ext1_SE]=MW[var]*10/AVO # convert [molec/cm2/s] to [kg/m2/s] - else: - dic_SE[var+'_CLXF']['PS'+ext1_SE]=0. - # End if - - # Aerosols - if var in AEROSOLS: - # for each species: - # original unit : [kg/kg] in dry air - # convert to [kg/m2] - # conversion: delP/gr - # delP will be applied when reading the files in SEbudget function. - if key in ListVars: - if var=='SO4': # For SO4, we report all the budget calculation for Sulfur. - dic_SE[var+'_BURDEN'][key+ext1_SE]=(32.066/115.11)/gr - else: - dic_SE[var+'_BURDEN'][key+ext1_SE]=1/gr - # End if - else: - dic_SE[var+'_BURDEN']['U'+ext1_SE]=0 - # End if - - - # for DDF: - # original unit: [kg/m2/s] - if key+'DDF' in ListVars: - if var=='SO4': - dic_SE[var+'_DDF'][key+ext1_SE+'DDF']=32.066/115.11 - else: - dic_SE[var+'_DDF'][key+ext1_SE+'DDF']=1 - # End if - else: - dic_SE[var+'_DDF']['PS'+ext1_SE]=0. - # End if - - - # for SFWET: - # original unit: [kg/m2/s] - if key+'SFWET' in ListVars: - if var=='SO4': - dic_SE[var+'_WDF'][key+ext1_SE+'SFWET']=32.066/115.11 - else: - dic_SE[var+'_WDF'][key+ext1_SE+'SFWET']=1 - # End if - else: - dic_SE[var+'_WDF']['PS'+ext1_SE]=0. - # End if - - - # for sfgaex1: - # original unit: [kg/m2/s] - if key+'_sfgaex1' in ListVars: - if var=='SO4': - dic_SE[var+'_GAEX'][key+ext1_SE+'_sfgaex1']=32.066/115.11 - else: - dic_SE[var+'_GAEX'][key+ext1_SE+'_sfgaex1']=1 - # End if - else: - dic_SE[var+'_GAEX']['PS'+ext1_SE]=0. - # End if - - - # for DDF in cloud water: - # original unit: [kg/m2/s] - cloud_key=key[:-2]+'c'+key[-1] - if cloud_key+ext1_SE+'DDF' in ListVars: - if var=='SO4': - dic_SE[var+'_DDFC'][cloud_key+ext1_SE+'DDF']=32.066/115.11 - else: - dic_SE[var+'_DDFC'][cloud_key+ext1_SE+'DDF']=1 - # End if - else: - dic_SE[var+'_DDFC']['PS'+ext1_SE]=0. - # End if - - # for SFWET in cloud water: - # original unit: [kg/m2/s] - if cloud_key+ext1_SE+'SFWET' in ListVars: - if var=='SO4': - dic_SE[var+'_WDFC'][cloud_key+ext1_SE+'SFWET']=32.066/115.11 - else: - dic_SE[var+'_WDFC'][cloud_key+ext1_SE+'SFWET']=1 - # End if - else: - dic_SE[var+'_WDFC']['PS'+ext1_SE]=0. - # End if - - if var=='SO4': - # for Nucleation : - # original unit: [kg/m2/s] - if key+ext1_SE+'_sfnnuc1' in ListVars: - dic_SE[var+'_NUCL'][key+ext1_SE+'_sfnnuc1']=32.066/115.11 - else: - dic_SE[var+'_NUCL']['PS'+ext1_SE]=0. - # End if - - # for Aqueous phase : - # original unit: [kg/m2/s] - if (('AQSO4_H2O2'+ext1_SE in ListVars) & ('AQSO4_O3'+ext1_SE in ListVars)) : - dic_SE[var+'_AQS']['AQSO4_H2O2'+ext1_SE]=32.066/115.11 - dic_SE[var+'_AQS']['AQSO4_O3'+ext1_SE]=32.066/115.11 - else: - # original unit: [kg/m2/s] - if cloud_key+'AQSO4'+ext1_SE in ListVars: - dic_SE[var+'_AQS'][cloud_key+'AQSO4'+ext1_SE]=32.066/115.11 - else: - dic_SE[var+'_AQS']['PS'+ext1_SE]=0. - # End if - - if cloud_key+'AQH2SO4'+ext1_SE in ListVars: - dic_SE[var+'_AQS'][cloud_key+'AQH2SO4'+ext1_SE]=32.066/115.11 - else: - dic_SE[var+'_AQS']['PS'+ext1_SE]=0. - # End if - # End if - # End if - - else: # Gases - # for each species: - # original unit : [mole/mole] in dry air - # convert to [kg/m2] - # conversion: Mw*delP/Mwair/gr Mwair=28.97 gr/mole - # delP will be applied when reading the files in SEbudget function. - if key in ListVars: - dic_SE[var+'_BURDEN'][key+ext1_SE]=MW[var]/Mwair/gr - else: - dic_SE[var+'_BURDEN']['U'+ext1_SE]=0 - # End if - - # for DF: - # original unit: [kg/m2/s] - if 'DF_'+key in ListVars: - dic_SE[var+'_DDF']['DF_'+key+ext1_SE]=1 - else: - dic_SE[var+'_DDF']['PS'+ext1_SE]=0. - # End if - - # for WD: - # original unit: [kg/m2/s] - if 'WD_'+key in ListVars: - dic_SE[var+'_WDF']['WD_'+key+ext1_SE]=1 - else: - dic_SE[var+'_WDF']['PS'+ext1_SE]=0. - # End if - - # for Chem tendency: - # original unit: [kg/s] - # conversion: not needed - if 'D'+key+'CHM' in ListVars: - dic_SE[var+'_TEND']['D'+key+'CHM'+ext1_SE]=1 # convert [kg/s] to [kg/s] - else: - dic_SE[var+'_TEND']['U'+ext1_SE]=0 - # End if - - # for Lightning NO production: (always in gas) - # original unit: [Tg N/Yr] - # conversion: not needed - if 'LNO_COL_PROD' in ListVars: - dic_SE[var+'_LNO']['LNO_COL_PROD'+ext1_SE]=1 # convert [Tg N/yr] to [Tg N /yr] - else: - dic_SE[var+'_LNO']['PS'+ext1_SE]=0 - # End if - # End if (aerosols or gases) - # End for - # End for - - # Write to log - adfobj.debug_log(msg) - - return dic_SE -##### - - -def make_Dic_scn_var_comp(adfobj, variables, current_dir, dic_SE, current_files, ext1_SE, AEROSOLS,Tropospheric): - """ - This function retrieves the files, latitude, and longitude information - in all the directories within the chosen dates. - - current_dir: list - - showing the directories to look for files. always end with '/' - - current_files: list - - List of CAM history files - - start_year: string - - Starting year - - end_year: string - - Ending year - - kwargs - ------ - ext1_SE: string - - specify if the files are for only a region, which changes to variable names. - ex: if you saved files for a only a box region ($LL_lat$,$LL_lon$,$UR_lat$,$UR_lon$), - the 'lat' variable will be saved as: 'lat_$LL_lon$e_to_$UR_lon$e_$LL_lat$n_to_$UR_lat$n' - for instance: 'lat_65e_to_91e_20n_to_32n' - - Returns - ------- - Dic_crit: - - dictionary for critical values for current case - Dic_scn_var_comp: - - full dictionary of all variables and components for current case - - NOTE: The LNO is lightning NOx, which should be reported explicitly rather as CO_LNO, O3_LNO, ... - """ - - # Set lists to gather necessary variables for logging - missing_vars_tot = [] - needed_vars_tot = [] - - # Initialize final component dictionary - Dic_var_comp={} - - for current_var in variables: - if 'AOD' in current_var: - components=[current_var+'_AOD'] - else: - if current_var in AEROSOLS: # AEROSOLS - - # Components are: burden, chemical loss, chemical prod, dry deposition, - # surface emissions, elevated emissions, wet deposition, gas-aerosol exchange - components=[current_var+'_BURDEN',current_var+'_CHML',current_var+'_CHMP', - current_var+'_DDF',current_var+'_WDF', current_var+'_SF', current_var+'_CLXF', - current_var+'_DDFC',current_var+'_WDFC'] - - if current_var=='SO4': - # For SULF we also have AQS, nucleation, and strat-trop gas exchange - components.append(current_var+'_AQS') - components.append(current_var+'_NUCL') - components.append(current_var+'_GAEX') - components.remove(current_var+'_CHMP') - - #components.append(current_var+'_CLXF') # BRT - CLXF is added above. - if current_var == "SOA": - components.append(current_var+'_GAEX') - #End if - AEROSOLS - - else: # CHEMS - # Components are: burden, chemical loss, chemical prod, dry/wet deposition, - # surface emissions, elevated emissions, chemical tendency - # I always add Lightning NOx production when calculating O3 budget. - - components=[current_var+'_BURDEN',current_var+'_CHML',current_var+'_CHMP', - current_var+'_DDF',current_var+'_WDF', current_var+'_SF', current_var+'_CLXF', - current_var+'_TEND'] - - if current_var =="O3": - components.append(current_var+'_LNO') - # End if - # End if - msg = f"chem/aerosol tables: 'make_Dic_scn_var_comp'" - msg += f"\n\t Current CAM variable: {current_var}" - msg += f"\n\t Derived components for CAM variable {current_var}: {components}" - #adfobj.debug_log(msg) - Dic_comp={} - Dic_comp,missing_vars,needed_vars=SEbudget(adfobj,dic_SE,current_dir,current_files,components,ext1_SE) - - for comp in components: - # Write details to log file - msg += f"\n\t\t calculate derived component: {comp} for main variable, {current_var}" - adfobj.debug_log(msg) - - # Gather info for debugging - for var_m in missing_vars: - if var_m not in missing_vars_tot: - missing_vars_tot.append(var_m) - for var_n in needed_vars: - if var_n not in needed_vars_tot: - needed_vars_tot.append(var_n) - # End for - # End for - # Set dictionary for key of current variable with dictionary values of all - # necessary constituents for calculating the current variable - Dic_var_comp[current_var] = Dic_comp - Dic_scn_var_comp = Dic_var_comp - - # Critical threshholds, just run this once - # this is for finding tropospheric values - # Critical threshholds?\n", - # Just run this once\n", - tropospheric_method='NA' - try: - current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['Pressure'],ext1_SE) - Dic_crit=current_crit['Pressure'] - tropospheric_method='pressure' - msg += f"\n\t WARNING: Troposphere is defined as Pressure>500 hPa" - except: - current_crit,_,__=SEbudget(adfobj,dic_SE,current_dir,current_files,['U'],ext1_SE) - Dic_crit=current_crit['U'] - Tropospheric=False - msg += f"\n\t WARNING: No way of defining troposphere was found in the model, budgets are total column" - # Log info to logging file - msg = f"chem/aerosol tables:" - msg += f"\n\t - potential missing variables from budget? {missing_vars_tot}" - adfobj.debug_log(msg) - - msg = f"chem/aerosol tables:" - msg += f"\n\t - needed variables for budget {needed_vars_tot}" - adfobj.debug_log(msg) - - return Dic_crit,Dic_scn_var_comp,Tropospheric,tropospheric_method -##### - - -def SEbudget(adfobj,dic_SE,data_dir,files,vars,ext1_SE,**kwargs): - """ - Function used for getting the data for the budget calculation. This is the - chunk of code that takes the longest by far. - - Example: - ** This is for both chemistry and aeorosl calculations - - dic_SE: dictionary specyfing what variables to get. For example, - for precipitation you can define SE as: - dic_SE['PRECT']={'PRECC'+ext1_SE:8.64e7,'PRECL'+ext1_SE:8.64e7} - - It means to sum the file variables "PRECC" and "PRECL" - for my arbitrary desired variable named "PRECT" - - - It also has the option to apply conversion factors. - For instance, PRECL and PRECC are in m/s. 8.64e7 is used to convernt m/s to mm/day - - - data_dir: string of the directory that contains the files. always end with '/' - - files: list of the files to be read - - var: string showing the variable to be extracted. - -> this will be the individual componnent, ie O3_CHMP, SOA_WDF, etc. - """ - - # gas constanct - Rgas=287.04 #[J/K/Kg]=8.314/0.028965 - - # Set lists to gather necessary variables for logging - missing_vars = [] - needed_vars = [] - Dic_all_data={} - -# all_data=[] - for file in range(len(files)): - - ds=xr.open_dataset(Path(data_dir) / files[file]) - - # Calculate these just once - if file==0: - mock_2d=np.zeros_like(np.array(ds['PS'+ext1_SE].isel(time=0))) - mock_3d=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) - - try: - delP=np.array(ds['PDELDRY'+ext1_SE].isel(time=0)) - except: - - hyai=np.array(ds['hyai']) - hybi=np.array(ds['hybi']) - - try: - PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) - except: - PS=np.array(ds['PS'+ext1_SE].isel(time=0)) - # End try/except - - P0=1e5 - Plevel=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) - - for i in range(len(Plevel)): - Plevel[i]=hyai[i]*P0+hybi[i]*PS - - delP=Plevel[1:]-Plevel[:-1] - - for var in vars: - if file == 0: - Dic_all_data[var]=[] - - - # Star gathering of variable data - - if var=='TROP_P': - data=np.array(ds['TROP_P'+ext1_SE].isel(time=0))/100 - elif var== 'Pressure': - try: - data=np.array(ds['PMID'+ext1_SE].isel(time=0))/100 - except: - hyam=np.array(ds['hyam']) - hybm=np.array(ds['hybm']) - try: - PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) - except: - PS=np.array(ds['PS'+ext1_SE].isel(time=0)) - P0=1e5 - data=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) - for i in range(len(data)): - data[i]=hyam[i]*P0+hybm[i]*PS - data=data/100 - else: - - - data=[] - for i in dic_SE[var].keys(): - - if file == 0: - msg = f"chem/aerosol tables: 'SEbudget'" - msg += f"\n\t\t ** variable(s) needed for derived var {var}: {dic_SE[var].keys()}" - msg += f"\n\t\t - constituent for derived var {var}: {i}" - adfobj.debug_log(msg) - if i not in needed_vars: - needed_vars.append(i) - if ((i!='PS'+ext1_SE) and (i!='U'+ext1_SE) ) : - data.append(np.array(ds[i].isel(time=0))*dic_SE[var][i]) - else: - if i=='PS'+ext1_SE: - data.append(mock_2d) - else: - data.append(mock_3d) - # End if - if file == 0: - - if var not in missing_vars: - if var!='U': # This is to avoid confusion between U variable or U mock! - missing_vars.append(var) - msg += f"\n\t\t - no variable was found for var {var}: {i}" - - # End if - - # Get total summed data for this history file data - data=np.sum(data,axis=0) - # End try/except - - if ('CHML' in var) or ('CHMP' in var) : - Temp=np.array(ds['T'+ext1_SE].isel(time=0)) - try: - Pres=np.array(ds['PMID'+ext1_SE].isel(time=0)) - except: - hyam=np.array(ds['hyam']) - hybm=np.array(ds['hybm']) - try: - PS=np.array(ds['PSDRY'+ext1_SE].isel(time=0)) - except: - PS=np.array(ds['PS'+ext1_SE].isel(time=0)) - P0=1e5 - Pres=np.zeros_like(np.array(ds['U'+ext1_SE].isel(time=0))) - for i in range(len(Pres)): - Pres[i]=hyam[i]*P0+hybm[i]*PS - rho= Pres/(Rgas*Temp) - data=data*delP/rho - elif ('BURDEN' in var): - data=data*delP - else: - data=data - # End if - # Add data to list - Dic_all_data[var].append(data) - ds.close() - for var in vars: # Take mean - Dic_all_data[var]=np.nanmean(Dic_all_data[var],axis=0) - - - return Dic_all_data,missing_vars,needed_vars -##### - - -def calc_budget_data(current_var, Dic_scn_var_comp, area, trop, inside, num_yrs, duration, AEROSOLS): - """ - Function to run through desired table values for calculations for the table entries - """ - - # Initialize full data dictionary for current table type - chem_dict = {} - - # Update variable marker if neccessary - if current_var == 'SO4': - specifier = ' S' - else: - specifier = '' - - # Calculate values for given variable - if 'AOD' in current_var: - # Burden - spc_burd = Dic_scn_var_comp[current_var][current_var+'_AOD'] - burden = np.ma.masked_where(inside==False,spc_burd) #convert Kg/m2 to Tg - BURDEN = np.ma.sum(burden*area)/np.ma.sum(area) - chem_dict[f"{current_var}_mean"] = np.round(BURDEN,5) - else: - # Surface Emissions - spc_sf = Dic_scn_var_comp[current_var][current_var+'_SF'] - tmp_sf = spc_sf - sf = np.ma.masked_where(inside==False,tmp_sf*area) #convert Kg/m2/s to Tg/yr - SF = np.ma.sum(sf*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_EMIS (Tg{specifier}/yr)"] = np.round(SF,5) - - # Elevated Emissions - spc_clxf = Dic_scn_var_comp[current_var][current_var+'_CLXF'] - tmp_clxf = spc_clxf - clxf = np.ma.masked_where(inside==False,tmp_clxf*area) #convert Kg/m2/s to Tg/yr - CLXF = np.ma.sum(clxf*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_EMIS_elevated (Tg{specifier}/yr)"] = np.round(CLXF,5) - - # Burden - spc_burd = Dic_scn_var_comp[current_var][current_var+'_BURDEN'] - spc_burd = np.where(np.isnan(trop),np.nan,spc_burd) - tmp_burden = np.nansum(spc_burd*area,axis=0) - burden = np.ma.masked_where(inside==False,tmp_burden) #convert Kg/m2 to Tg - BURDEN = np.ma.sum(burden*1e-9) - chem_dict[f"{current_var}_BURDEN (Tg{specifier})"] = np.round(BURDEN,5) - - # Chemical Loss - spc_chml = Dic_scn_var_comp[current_var][current_var+'_CHML'] - spc_chml = np.where(np.isnan(trop),np.nan,spc_chml) - tmp_chml = np.nansum(spc_chml*area,axis=0) - chml = np.ma.masked_where(inside==False,tmp_chml) #convert Kg/m2/s to Tg/yr - CHML = np.ma.sum(chml*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_CHEM_LOSS (Tg{specifier}/yr)"] = np.round(CHML,5) - - # Chemical Production - if current_var == 'SO4': # chemical production is basically the elevated emissions. - # We have removed it for SO4 budget. and put 0 here, so, we don't report it - chem_dict[f"{current_var}_CHEM_PROD (Tg{specifier}/yr)"] = 0 - else: - spc_chmp = Dic_scn_var_comp[current_var][current_var+'_CHMP'] - spc_chmp = np.where(np.isnan(trop),np.nan,spc_chmp) - tmp_chmp = np.nansum(spc_chmp*area,axis=0) - chmp = np.ma.masked_where(inside==False,tmp_chmp) #convert Kg/m2/s to Tg/yr - CHMP = np.ma.sum(chmp*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_CHEM_PROD (Tg{specifier}/yr)"] = np.round(CHMP,5) - # End if - - # Aerosol calculations - #--------------------- - if current_var in AEROSOLS: - - # Dry Deposition Flux - spc_ddfa = Dic_scn_var_comp[current_var][current_var+'_DDF'] - spc_ddfc = Dic_scn_var_comp[current_var][current_var+'_DDFC'] - spc_ddf = spc_ddfa +spc_ddfc - tmp_ddf = spc_ddf - ddf = np.ma.masked_where(inside==False,tmp_ddf*area) #convert Kg/m2/s to Tg/yr - DDF = np.ma.sum(ddf*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_DRYDEP (Tg{specifier}/yr)"] = np.round(DDF,5) - - # Wet deposition - spc_wdfa = Dic_scn_var_comp[current_var][current_var+'_WDF'] - spc_wdfc = Dic_scn_var_comp[current_var][current_var+'_WDFC'] - spc_wdf = spc_wdfa +spc_wdfc - tmp_wdf = spc_wdf - wdf = np.ma.masked_where(inside==False,tmp_wdf*area) #convert Kg/m2/s to Tg/yr - WDF = -1*np.ma.sum(wdf*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_WETDEP (Tg{specifier}/yr)"] = np.round(WDF,5) - - if current_var in ["SOA",'SO4']: - # gas-aerosol Exchange - spc_gaex = Dic_scn_var_comp[current_var][current_var+'_GAEX'] - tmp_gaex = spc_gaex - gaex = np.ma.masked_where(inside==False,tmp_gaex*area) #convert Kg/m2/s to Tg/yr - GAEX = np.ma.sum(gaex*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_GAEX (Tg{specifier}/yr)"] = np.round(GAEX,5) - - # LifeTime = Burden/(loss+deposition) no chemical loss for aerosols - LT = BURDEN/(DDF+WDF)* duration/86400/num_yrs # days - chem_dict[f"{current_var}_LIFETIME (days)"] = np.round(LT,5) - - if current_var == 'SO4': - # Aqueous Chemistry - spc_aqs = Dic_scn_var_comp[current_var][current_var+'_AQS'] - tmp_aqs = spc_aqs - aqs = np.ma.masked_where(inside==False,tmp_aqs*area) #convert Kg/m2/s to Tg/yr - AQS = np.ma.sum(aqs*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_AQUEOUS (Tg{specifier}/yr)"] = np.round(AQS,5) - - # Nucleation - spc_nucl = Dic_scn_var_comp[current_var][current_var+'_NUCL'] - tmp_nucl = spc_nucl - nucl = np.ma.masked_where(inside==False,tmp_nucl*area) #convert Kg/m2/s to Tg/yr - NUCL = np.ma.sum(nucl*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_NUCLEATION (Tg{specifier}/yr)"] = np.round(NUCL,5) - - # Gaseous calculations - #--------------------- - else: - # Dry Deposition Flux - spc_ddf = Dic_scn_var_comp[current_var][current_var+'_DDF'] - tmp_ddf = spc_ddf - ddf = np.ma.masked_where(inside==False,tmp_ddf*area) #convert Kg/m2/s to Tg/yr - DDF = np.ma.sum(ddf*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_DRYDEP (Tg/yr)"] = np.round(DDF,5) - - # Wet Deposition Flux - spc_wdf = Dic_scn_var_comp[current_var][current_var+'_WDF'] - tmp_wdf = spc_wdf - wdf = np.ma.masked_where(inside==False,tmp_wdf*area) #convert Kg/m2/s to Tg/yr - WDF = -1*np.ma.sum(wdf*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_WETDEP (Tg/yr)"] = np.round(WDF,5) - - # Total Deposition - TDEP = DDF + WDF - chem_dict[f"{current_var}_TDEP (Tg/yr)"] = np.round(TDEP,5) - - # LifeTime = Burden/(loss+deposition) - if current_var == "CH4": - LT = BURDEN/(CHML+DDF+WDF) # years - chem_dict[f"{current_var}_LIFETIME (years)"] = np.round(LT,5) - else: - if (CHML+DDF+WDF) > 0: - if CHML != 0: - LT = BURDEN/(CHML+DDF+WDF)*duration/86400/num_yrs # days - chem_dict[f"{current_var}_LIFETIME (days)"] = np.round(LT,5) - else: - # do not report lifetime if chemical loss (for gases) is not included in the model outputs - # and put 0 here, so, we don't report it - chem_dict[f"{current_var}_LIFETIME (days)"] = 0 - # End if - # End if - # End if - - #NET = CHMP-CHML - # Chemical Tendency - spc_tnd = Dic_scn_var_comp[current_var][current_var+'_TEND'] - spc_tnd = np.where(np.isnan(trop),np.nan,spc_tnd) - tmp_tnd = np.nansum(spc_tnd,axis=0) - tnd = np.ma.masked_where(inside==False,tmp_tnd) #convert Kg/s to Tg/yr - TND = np.ma.sum(tnd*duration*1e-9)/num_yrs - chem_dict[f"{current_var}_TEND (Tg/yr)"] = np.round(TND,5) - - # O3 dependent calculations - if current_var == "O3": - # Stratospheric-Tropospheric Exchange - STE = DDF-TND - chem_dict[f"{current_var}_STE (Tg/yr)"] = np.round(STE,5) - - # Lightning NOX production - spc_lno = Dic_scn_var_comp[current_var][current_var+'_LNO'] - tmp_lno = np.ma.masked_where(inside==False,spc_lno) - LNO = np.ma.sum(tmp_lno) - chem_dict[f"{current_var}_LNO (Tg N/yr)"] = np.round(LNO,5) - # End if (aerosol or gas) - return chem_dict -##### - - -def make_table(adfobj, vars, chem_type, Dic_scn_var_comp, areas, trops, case_names, nicknames, durations, insides, num_yrs, AEROSOLS): - """ - Create CSV table for aeorosols and gases, if applicable - - Table includes column values of variable, case(s), difference (if applicable) - - If this is a single model vs model run: 4 columns - first column: variables names, - second column: test case variable values - third column: baseline case variable values - final column: difference of test and baseline. - If this is a model vs obs run: 2 columns - first column: variables names, - second column: test case variable values - """ - # Initialize an empty dictionary to store DataFrames - dfs = {} - - #Special ADF variable which contains the output paths for - #all generated plots and tables for each case: - output_locs = adfobj.plot_location - - #Convert output location string to a Path object: - output_location = Path(output_locs[0]) - - # Loop over model cases - - for i,case in enumerate(nicknames): - - nickname = case - - # Collect row data in a list of dictionaries - #durations[case] - rows = [] - for current_var in vars: - chem_dict = calc_budget_data(current_var, Dic_scn_var_comp[case], areas[case], trops[case], insides[case], - num_yrs[case], durations[case], AEROSOLS) - # Loop through table variables - for key, val in chem_dict.items(): - if val != 0: # Skip variables with a value of 0 - print(f"\t - Variable '{key}' being added to table") - rows.append({'variable': key, nickname: np.round(val, 3)}) - elif 'OASISS_EMIS (' in key: # the paranthesis is to ignore EMIS_Elevated variables! - print(f"\t - Variable '{key}' being added to table") - rows.append({'variable': key, nickname: np.round(val, 3)}) - else: - msg = f"chem/aerosol tables:" - msg += f"\n\t - Variable '{key}' has value of 0, will not add to table" - adfobj.debug_log(msg) - # End if - # End for - # End for - - # Create the DataFrame for the current case - table_df = pd.DataFrame(rows) - - if chem_type == 'gases': - # Replace compound names directly in the DataFrame - replacements = { - 'MTERP': 'Monoterpene', - 'CH3OH': 'Methanol', - 'CH3COCH3': 'Acetone', - 'O3_LNO': 'LNOx_PROD' - } - table_df['variable'] = table_df['variable'].replace(replacements, regex=True) - # End if - - # Store the DataFrame in the dictionary - dfs[nickname] = table_df - - # End for - - # Merge the DataFrames on the 'variable' column - if len(case_names) == 2: - - table_df = pd.merge(dfs[nicknames[0]], dfs[nicknames[1]], on='variable') - - # Calculate the differences between case columns - table_df['difference'] = table_df[nicknames[0]] - table_df[nicknames[1]] - - #Create output file name: - output_csv_file = output_location / f'ADF_amwg_{chem_type}_table.csv' - # Save table to CSV and add table dataframe to website (if enabled) - table_df.to_csv(output_csv_file, index=False) - #adfobj.add_website_data(table_df, chem_type, case, plot_type="Tables") - adfobj.add_website_data(table_df, chem_type, case_names[0], plot_type="Tables") - -##### From 3ae6961beb460c7930522b7201e2d31c87e5816f Mon Sep 17 00:00:00 2001 From: justin-richling Date: Tue, 9 Sep 2025 10:49:52 -0600 Subject: [PATCH 53/91] Pull time stamp info out of if statment Since the run info needs the same time stamp as the log file, make this happen regardless if debug is true --- lib/adf_base.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/adf_base.py b/lib/adf_base.py index 64618f2c0..e0f2ebe54 100644 --- a/lib/adf_base.py +++ b/lib/adf_base.py @@ -49,16 +49,16 @@ def __init__(self, debug = False): raise TypeError("'debug' must be a boolean type (True or False)") self.__debug_fname = '' + # Get the current date and time + current_timestamp = datetime.now() + # Format the datetime object to a string without microseconds + dt_str = current_timestamp.strftime('%Y-%m-%d %H:%M:%S') + ext = f'{str(dt_str).replace(" ","-")}' + debug_fname = f"ADF_debug_{ext}.log" + self.__debug_fname = debug_fname # Create debug log, if requested: if debug: - # Get the current date and time - current_timestamp = datetime.now() - # Format the datetime object to a string without microseconds - dt_str = current_timestamp.strftime('%Y-%m-%d %H:%M:%S') - ext = f'{str(dt_str).replace(" ","-")}' - debug_fname = f"ADF_debug_{ext}.log" - self.__debug_fname = debug_fname logging.basicConfig(filename=debug_fname, level=logging.DEBUG) self.__debug_log = logging.getLogger("ADF") else: @@ -102,4 +102,4 @@ def end_diag_fail(self, msg: str): #++++++++++++++++++++ #End Class definition -#++++++++++++++++++++ +#++++++++++++++++++++ \ No newline at end of file From 7a18bb48bb7d22e7b8ddcf1cc942a8c243ef68da Mon Sep 17 00:00:00 2001 From: justin-richling Date: Tue, 9 Sep 2025 10:57:30 -0600 Subject: [PATCH 54/91] Add method to collect current git info --- lib/adf_config.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/lib/adf_config.py b/lib/adf_config.py index 61a8ba112..625a67c46 100644 --- a/lib/adf_config.py +++ b/lib/adf_config.py @@ -19,6 +19,8 @@ import os.path import re import copy +import os +import subprocess #+++++++++++++++++++++++++++++++++++++++++++++++++ #import non-standard python modules, including ADF @@ -308,6 +310,60 @@ def read_config_var(self, varname, conf_dict=None, required=False): #config variables dictionary: return copy.deepcopy(var) + def config_dict(self): + config_dict = self.__config_dict + return copy.copy(config_dict) + + def get_git_info(self): + + """ + Gather currnet Git info during ADF run. + + Returns: + -------- + info : dict + Dictionary with the following keys: + - branch: Current Git branch name. + - commit: Current commit hash. + - remote_url: URL of the remote repository. + - repo_name: Name of the repository. + - is_dirty: Boolean indicating if there are uncommitted changes. + """ + + #Initialize empty dictionary: + info = {} + + try: + # Current branch + branch = subprocess.run(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], + stdout=subprocess.PIPE, text=True, check=True).stdout.strip() + info['branch'] = branch + + # Commit hash + commit = subprocess.run(['git', 'rev-parse', 'HEAD'], + stdout=subprocess.PIPE, text=True, check=True).stdout.strip() + info['commit'] = commit + + # Remote URL + remote_url = subprocess.run(['git', 'remote', 'get-url', 'origin'], + stdout=subprocess.PIPE, text=True, check=True).stdout.strip() + info['remote_url'] = remote_url + + # Repo name + info['repo_name'] = os.path.splitext(os.path.basename(remote_url))[0] + + # Status + status = subprocess.run(['git', 'status', '--short'], + stdout=subprocess.PIPE, text=True, check=True).stdout.strip() + info['is_dirty'] = bool(status) + + except subprocess.CalledProcessError as e: + print("Git command failed:", e) + info = None + + return info + ######### + #++++++++++++++++++++ #End Class definition -#++++++++++++++++++++ +#++++++++++++++++++++ \ No newline at end of file From 9c438e76ed29f86356cfabf36332c48443e4818f Mon Sep 17 00:00:00 2001 From: justin-richling Date: Tue, 9 Sep 2025 11:06:34 -0600 Subject: [PATCH 55/91] Add code to gather ADF run info --- lib/adf_info.py | 109 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/lib/adf_info.py b/lib/adf_info.py index 41ebde606..5bbc0cc41 100644 --- a/lib/adf_info.py +++ b/lib/adf_info.py @@ -32,6 +32,7 @@ import copy import os import getpass +import subprocess #+++++++++++++++++++++++++++++++++++++++++++++++++ #import non-standard python modules, including ADF @@ -616,6 +617,86 @@ def __init__(self, config_file, debug=False): self.debug_log(f"ADF is running with {self.__num_procs} processors.") # ----------------------------------------- + active_env = self.get_active_conda_environment() + if not active_env: + active_env = "--" + + # Gather ADF run env info + log_name = self.debug_fname() + #Create new path object from user-specified plot directory path: + plot_path = Path(self.plot_location[0]) + + #Create directory path where the website will be built: + website_dir = plot_path / "website" + Path(website_dir).mkdir(parents=True, exist_ok=True) + + self.__run_info = f"{log_name}".replace("debug","run_info").replace(".log",".md") + run_info = f"{website_dir}/{self.__run_info}" + + four_space = "    " + two_space = "  " + font_22 = "style='font-size:22px;'" + font_18 = "style='font-size:18px;'" + font_16 = "style='font-size:16px;'" + + with open(run_info, "w") as f: + log_msg = f"adf_info: ADF run info:" + + # Gather config yaml file info + config_file_msg = "\nConfig file used:" + msg = f"{config_file_msg}\n{'-' * (len(config_file_msg))}\n {config_file}\n" + log_msg += msg + + f.write("

") + f.write(f"Config file used
") + f.write(f"{two_space}{config_file}

") + + config_msg = "\n Config file options:" + msg = f"{config_msg}\n {'- ' * (int(len(config_msg)/2)-1)}" + log_msg += msg + + f.write(f" Config file options
") + for key,val in AdfConfig.config_dict(self).items(): + if isinstance(val,dict): + log_msg += f"\n {key}:" + f.write(f"{two_space}{key}:
") + for key2,val2 in val.items(): + log_msg += f"\n {key2}: {val2}" + f.write(f"{four_space}{key2}: {val2}
") + elif isinstance(val,list): + f.write(f"{two_space}{key}:
") + log_msg += f"\n {key}:" + for val2 in val: + log_msg += f"\n {val2}" + f.write(f"{four_space}{val2}
") + else: + f.write(f"{two_space}{key}: {val}
") + log_msg += f"\n {key}: {val}" + + # Gather Conda environment + conda_msg = "\nConda env used:" + msg = f"{conda_msg}\n{'-' * (len(conda_msg)-1)}\n" + log_msg += f"\n {msg}" + + f.write(f"\n") + f.write(f"
Conda env used
") + f.write(f"{two_space}{active_env}") + log_msg += f" {active_env}" + + git_info = self.get_git_info() + git_msg = "\nGit Info:" + msg = f"{git_msg}\n{'-' * (len(git_msg)-1)}\n" + log_msg += f"\n {msg}" + + # Gather Git info + f.write(f"\n") + f.write(f"

Git Info
") + for key,val in git_info.items(): + log_msg += f" {key}: {val}\n" + f.write(f"{two_space}{key}: {val}
") + f.write("

") + + self.debug_log(log_msg) ######### def hist_str_to_list(self, conf_var, conf_val): """ @@ -728,6 +809,11 @@ def hist_string(self): base_hist_strs = "" hist_strs = {"test_hist_str":cam_hist_strs, "base_hist_str":base_hist_strs} return hist_strs + + @property + def run_info(self): + run_info = self.__run_info + return run_info ######### @@ -898,6 +984,29 @@ def get_climo_yrs_from_ts(self, input_ts_loc, case_name): print(msg) return syr, eyr + + + def get_active_conda_environment(self): + """ + Utility function to get the name of the active conda environment. + + Returns: + -------- + env_name (str or None): Name of the active conda environment, or None if not found. + """ + env_name = None + try: + # Execute 'conda env list' and capture output + result = subprocess.run(['conda', 'env', 'list'], capture_output=True, text=True, check=True) + output_lines = result.stdout.splitlines() + for i,line in enumerate(output_lines): + # The active environment is marked with an asterisk (*) + if '*' in line.strip(): + # Extract the environment name (first part of the line) + env_name = line.strip().split()[0] + except subprocess.CalledProcessError as e: + print(f"Error executing conda command: {e}") + return env_name #++++++++++++++++++++ #End Class definition From ad8f3a8eeecfa94aace5987d9b2ebad8531a47e6 Mon Sep 17 00:00:00 2001 From: justin-richling Date: Tue, 9 Sep 2025 11:10:48 -0600 Subject: [PATCH 56/91] Create ADF run info html page --- lib/adf_web.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/adf_web.py b/lib/adf_web.py index e39981f8f..0ae3293f1 100644 --- a/lib/adf_web.py +++ b/lib/adf_web.py @@ -22,6 +22,7 @@ import os import os.path +import markdown from pathlib import Path @@ -695,6 +696,33 @@ def jinja_enumerate(arg): #Also check if index page exists for this case: index_html_file = \ self.__case_web_paths[web_data.case]['website_dir'] / "index.html" + print("index_html_file",index_html_file) + + # Create run info web page + run_info_md_file = \ + self.__case_web_paths[web_data.case]['website_dir'] / self.run_info + print("run_info_md_file",run_info_md_file) + + # Read the markdown file + with open(run_info_md_file, "r", encoding="utf-8") as mdfile: + md_text = mdfile.read() + + # Convert markdown to HTML + run_info_html = markdown.markdown(md_text) + index_title = "AMP Diagnostics Prototype" + run_info_html_file = self.__case_web_paths[web_data.case]['website_dir'] / "run_info.html" + run_info_tmpl = jinenv.get_template('template_run_info.html') + run_info_rndr = run_info_tmpl.render(run_info=run_info_html, + title=index_title, + case_name=web_data.case, + base_name=data_name, + case_yrs=case_yrs, + baseline_yrs=baseline_yrs, + plot_types=plot_types, + run_info=run_info_html_file) + + with open(run_info_html_file, "w", encoding="utf-8") as htmlfile: + htmlfile.write(run_info_rndr) #Re-et plot types list: if web_data.case == 'multi-case': @@ -728,7 +756,8 @@ def jinja_enumerate(arg): plot_types=plot_types, avail_plot_types=avail_plot_types, avail_external_packages=avail_external_packages, - external_package_links=self.external_package_links) + external_package_links=self.external_package_links, + run_info=run_info_html_file) #Write Mean diagnostics index HTML file: with open(index_html_file, 'w', encoding='utf-8') as ofil: From d13c1e07f68fe9faddaae8539597c2ce2a6ea48d Mon Sep 17 00:00:00 2001 From: justin-richling Date: Tue, 9 Sep 2025 11:13:55 -0600 Subject: [PATCH 57/91] update html templates to add run info --- lib/website_templates/template.html | 1 + lib/website_templates/template_index.html | 1 + lib/website_templates/template_mean_diag.html | 1 + lib/website_templates/template_mean_tables.html | 1 + lib/website_templates/template_multi_case_index.html | 1 + lib/website_templates/template_table.html | 1 + 6 files changed, 6 insertions(+) diff --git a/lib/website_templates/template.html b/lib/website_templates/template.html index 3a06250cb..61c4b2a80 100644 --- a/lib/website_templates/template.html +++ b/lib/website_templates/template.html @@ -12,6 +12,7 @@