diff --git a/.github/workflows/build_mpas.yml b/.github/workflows/build_mpas.yml index 4fb3150d19..f8aa32125c 100644 --- a/.github/workflows/build_mpas.yml +++ b/.github/workflows/build_mpas.yml @@ -1,6 +1,7 @@ name: Build (GNU) MPAS Standalone -on: [pull_request,workflow_dispatch] +#on: [pull_request,workflow_dispatch] +on: workflow_dispatch jobs: build_mpas_GNU: @@ -37,7 +38,7 @@ jobs: - name: Install NetCDF library run: | sudo apt-get update - sudo apt-get install libnetcdff-dev + sudo apt-get install libnetcdff-dev liblapack-dev libblas-dev - name: Cache openmpi id: cache-openmpi @@ -138,6 +139,18 @@ jobs: echo "LD_LIBRARY_PATH=${PNETCDF}/bin:$LD_LIBRARY_PATH" >> $GITHUB_ENV echo "PATH=${PNETCDF}/bin:$PATH" >> $GITHUB_ENV + - name: Setup NetCDF + run: | + NETCDF_DIR=/home/runner/NetCDF + mkdir -p ${NETCDF_DIR}/lib ${NETCDF_DIR}/include + for f in /usr/include/netcdf*.h /usr/include/netcdf*.inc /usr/include/netcdf*.mod; do + [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/include/ || true + done + for f in /usr/lib/x86_64-linux-gnu/libnetcdf*.so; do + [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/lib/ || true + done + echo "NETCDF=${NETCDF_DIR}" >> $GITHUB_ENV + # - name: Setup environment # run: | # echo "FC=mpif90" >> $GITHUB_ENV diff --git a/.github/workflows/build_mpas_intel.yml b/.github/workflows/build_mpas_intel.yml index f7e1a46c9d..7911d33313 100644 --- a/.github/workflows/build_mpas_intel.yml +++ b/.github/workflows/build_mpas_intel.yml @@ -1,6 +1,7 @@ name: Build (Intel) MPAS Standalone -on: [pull_request,workflow_dispatch] +#on: [pull_request,workflow_dispatch] +on: workflow_dispatch jobs: build_mpas_intel: @@ -41,6 +42,7 @@ jobs: install-mpi: true install-oneapi: false mpi-wrapper-setup: classic + install-mkl: true - name: Cache Intel id: cache-INTEL @@ -49,10 +51,17 @@ jobs: path: /opt/intel/oneapi/mpi/2021.10.0 key: cache-AUTOTOOLS-${{matrix.fortran-compiler}}-key - - name: Install NetCDF library + - name: Install NetCDF-C library run: | sudo apt-get update - sudo apt-get install libnetcdff-dev + sudo apt-get install libnetcdf-dev + + - name: Cache NetCDF-Fortran + id: cache-netcdf-fortran + uses: actions/cache@v4 + with: + path: /home/runner/NetCDF + key: cache-NetCDF-Fortran-${{matrix.fortran-compiler}}-key - name: Cache GNU autotools id: cache-AUTOTOOLS @@ -102,7 +111,7 @@ jobs: key: cache-PnetCDF-${{matrix.fortran-compiler}}-key - name: Install PnetCDF - if: steps.cache-PnetCDF.outputs.cache-hit != 'true' + if: steps.cache-pnetcdf.outputs.cache-hit != 'true' run: | set -x echo "LD_LIBRARY_PATH=${AUTOTOOLS}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV @@ -129,6 +138,23 @@ jobs: echo "LD_LIBRARY_PATH=${AUTOTOOLS}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV echo "PATH=${PNETCDF}/bin:$PATH" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=${PNETCDF}/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "NETCDF=/home/runner/NetCDF" >> $GITHUB_ENV + + - name: Build NetCDF-Fortran + if: steps.cache-netcdf-fortran.outputs.cache-hit != 'true' + run: | + NETCDF_F_VERSION=4.6.1 + wget -q https://github.com/Unidata/netcdf-fortran/archive/refs/tags/v${NETCDF_F_VERSION}.tar.gz + tar -xf v${NETCDF_F_VERSION}.tar.gz + cd netcdf-fortran-${NETCDF_F_VERSION} + autoreconf -i + FC=mpiifort CC=mpiicc \ + CPPFLAGS="-I$(nc-config --includedir)" \ + LDFLAGS="-L$(nc-config --libdir)" \ + ./configure --prefix=/home/runner/NetCDF + make -j 8 install + NCLIBDIR=$(nc-config --libdir) + ln -sf ${NCLIBDIR}/libnetcdf.so /home/runner/NetCDF/lib/libnetcdf.so - name: Build MPAS Standalone (make) if: matrix.build-system == 'make' @@ -144,4 +170,4 @@ jobs: mkdir build cd build cmake -DMPAS_DOUBLE_PRECISION=OFF -DMPAS_CORES="init_atmosphere;atmosphere" .. - make -j8 \ No newline at end of file + make -j8 diff --git a/.github/workflows/run_mpas.yml b/.github/workflows/run_mpas.yml index f4e911f4d8..fbf48ef208 100644 --- a/.github/workflows/run_mpas.yml +++ b/.github/workflows/run_mpas.yml @@ -1,6 +1,7 @@ name: Run MPAS Standalone (NOAA GSL tests) -on: [push, pull_request, workflow_dispatch] +#on: [push, pull_request, workflow_dispatch] +on: workflow_dispatch ############################################################################################# # Testing script/workflow for ufs-community fork of MPAS-Dev/MPAS-A. @@ -98,6 +99,23 @@ jobs: echo "FC=mpiifx" >> $GITHUB_ENV echo "MPICC=/opt/intel/oneapi/mpi/latest/bin/mpiicx" >> $GITHUB_ENV + - name: Install LAPACK/BLAS + run: | + apt-get update -y + apt-get install -y liblapack-dev libblas-dev libnetcdff-dev + + - name: Setup NetCDF + run: | + NETCDF_DIR=/home/runner/NetCDF + mkdir -p ${NETCDF_DIR}/lib ${NETCDF_DIR}/include + for f in /usr/include/netcdf*.h /usr/include/netcdf*.inc /usr/include/netcdf*.mod; do + [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/include/ || true + done + for f in /usr/lib/x86_64-linux-gnu/libnetcdf*.so; do + [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/lib/ || true + done + echo "NETCDF=${NETCDF_DIR}" >> $GITHUB_ENV + - name: Setup PnetCDF run: | echo "PNETCDF=/opt/pnetcdf" >> $GITHUB_ENV diff --git a/.github/workflows/run_mpas_hrrr.yml b/.github/workflows/run_mpas_hrrr.yml index 655c70b6ec..9ae23038ce 100644 --- a/.github/workflows/run_mpas_hrrr.yml +++ b/.github/workflows/run_mpas_hrrr.yml @@ -1,6 +1,7 @@ name: Run MPAS Standalone (NOAA GSL HRRRv5 tests) -on: [push, pull_request, workflow_dispatch] +#on: [push, pull_request, workflow_dispatch] +on: workflow_dispatch ############################################################################################# # Testing script/workflow for ufs-community fork of MPAS-Dev/MPAS-A. @@ -79,6 +80,23 @@ jobs: echo "CXX=mpicxx" >> $GITHUB_ENV echo "FC=mpif90" >> $GITHUB_ENV + - name: Install LAPACK/BLAS + run: | + apt-get update -y + apt-get install -y liblapack-dev libblas-dev libnetcdff-dev + + - name: Setup NetCDF + run: | + NETCDF_DIR=/home/runner/NetCDF + mkdir -p ${NETCDF_DIR}/lib ${NETCDF_DIR}/include + for f in /usr/include/netcdf*.h /usr/include/netcdf*.inc /usr/include/netcdf*.mod; do + [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/include/ || true + done + for f in /usr/lib/x86_64-linux-gnu/libnetcdf*.so; do + [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/lib/ || true + done + echo "NETCDF=${NETCDF_DIR}" >> $GITHUB_ENV + - name: Setup PnetCDF run: | echo "PNETCDF=/opt/pnetcdf" >> $GITHUB_ENV diff --git a/.github/workflows/run_mpas_stoch.yml b/.github/workflows/run_mpas_stoch.yml new file mode 100644 index 0000000000..0a38093f18 --- /dev/null +++ b/.github/workflows/run_mpas_stoch.yml @@ -0,0 +1,405 @@ +name: Run MPAS Standalone with Stochastic Physics (NOAA GSL HRRRv5 tests) + +on: [push, pull_request, workflow_dispatch] +#on: workflow_dispatch + +############################################################################################# +# Testing script/workflow for ufs-community fork of MPAS-Dev/MPAS-A. +# +# Description +# +# This workflow will run MPAS-Model with stochastic physics enabled in order to compare +# output from the baseline branch to one from the feature branch. +# +# - Clone and build MPAS for baseline generation. +# - Clone and build MPAS for feature branch testing (against baselines). +# - Download any data (MP tables, MPAS ICs/LBCs/etc...). +# - Create/populate MPAS run directories. +# - Run MPAS using baseline codebase/configuration. +# - Run MPAS using feature branch. +# - Compare results. +# - Save output files to GitHub artifact (for comparision w/ inline/standaloen results) +# +# Comments: +# - The test build/run configurations matrix is described below. +# - This script uses a matrix run configuration with exclusions to achieve the desired "run list" +# +# Tests: +# Baseline Codebase Repository:Branch Physics IC source season build-type +# 1) ufs-community:noaa/develop hrrrv5 gfs winter Release +# +############################################################################################# +jobs: + run_mpas_stoch: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false # Disable fail-fast + matrix: + f-compiler: [gfortran]#,ifx] +# ic_source: [gfs, rap] + ic_source: [gfs] +# season: [summer, winter] + season: [winter] + build-type: [Release] + physics: [hrrrv5] + exclude: + - ic_source: gfs + season: summer + include: + # Set container images for each compiler + - f-compiler: gfortran + bld_target: gfortran + image: dustinswales/ufs-community-mpas-ci:gnu +# - f-compiler: ifx +# bld_target: intel +# image: dustinswales/ufs-community-mpas-ci:oneapi + container: + image: ${{matrix.image}} + # + defaults: + run: + shell: bash -el {0} + + # Environmental variables. GitHub Actions automatically exports these to all shell + # environments. + env: + py-version: 3.11 + runner_ROOT: /__w + PNETCDF: /opt/pnetcdf + ufs_community: testing_and_setup/ufs-community + # Github artifact name. + artifact: artifact-${{matrix.f-compiler}}-${{matrix.build-type}}-${{matrix.ic_source}}-${{matrix.season}}-${{matrix.physics}}-stoch + + # Workflow steps + steps: + ########################################################################################## + # Setup + ########################################################################################## + - name: Set derived shell environment variables + run: | + # + # Define some relative paths. These are with respect to the root directory of the + # MPAS-Model clone (which may be SRCDIR_BL or SRCDIR_RF). + # + # The wrf_phys_tables directory contains physics tables and other files specific to + # WRF physics. + echo "wrf_phys_tables=src/core_atmosphere/physics/physics_wrf/files" >> $GITHUB_ENV + # The ufscmnty_cases directory contains MPAS configuration files for various CI test + # cases. These consist of MPAS namelists and stream definitions. + echo "ufscmnty_cases=${ufs_community}/cases" >> $GITHUB_ENV + # The ufscmnty_data directory contains the script (get_data.sh) that downloads into + # subdirectories various files needed for running MPAS. These include physics tables + # specific to UFS physics parameterizations (as opposed to WRF physics), MPAS initial + # and boundary conditions, and MPAS static files. + ufscmnty_data="${ufs_community}/data" + echo "ufscmnty_data=${ufscmnty_data}" >> $GITHUB_ENV + # The ufs_phys_tables directory contains physics tables and other files specific to + # ufs physics. + echo "ufs_phys_tables=${ufscmnty_data}/tables" >> $GITHUB_ENV + # The mpas_ics directory contains MPAS initial and boundary condition files and MPAS + # static files. + echo "mpas_ics=${ufscmnty_data}/ics" >> $GITHUB_ENV + # + # Define base directories for git clones of the MPAS baseline and feature branches. + # + echo "SRCDIR_BL=/__w/MPAS-Model-BL" >> $GITHUB_ENV + # Note that GITHUB_WORKSPACE is set using the pattern + # /__w// + # i.e. the repository name is repeated twice: once for the parent directory and once + # for the checkout directory within it. Thus, here, it will be set to + # /__w/MPAS-Model/MPAS-Model + # and SRCDIR_FT will be set to that as well. + echo "SRCDIR_FT=${GITHUB_WORKSPACE}" >> $GITHUB_ENV + # Define absolute paths to the base directories of the runs with the baseline and feature + # branches. + # + echo "RUNDIR_BL=${runner_ROOT}/run_bl" >> $GITHUB_ENV + echo "RUNDIR_FT=${runner_ROOT}/run_ft" >> $GITHUB_ENV + # + # Set the absolute path to the Github artifact. + # + echo "ARTIFACT_DIR=${runner_ROOT}/${artifact}" >> $GITHUB_ENV + + - name: Setup MPI (GNU) + if: matrix.f-compiler == 'gfortran' + run: | + echo "CC=mpicc" >> $GITHUB_ENV + echo "CXX=mpicxx" >> $GITHUB_ENV + echo "FC=mpif90" >> $GITHUB_ENV + + - name: Install LAPACK/BLAS + run: | + apt-get update -y + apt-get install -y liblapack-dev libblas-dev libnetcdff-dev + + - name: Setup NetCDF + run: | + NETCDF_DIR=/home/runner/NetCDF + mkdir -p ${NETCDF_DIR}/lib ${NETCDF_DIR}/include + for f in /usr/include/netcdf*.h /usr/include/netcdf*.inc /usr/include/netcdf*.mod; do + [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/include/ || true + done + for f in /usr/lib/x86_64-linux-gnu/libnetcdf*.so; do + [ -e "$f" ] && ln -sf "$f" ${NETCDF_DIR}/lib/ || true + done + echo "NETCDF=${NETCDF_DIR}" >> $GITHUB_ENV + + - name: Setup PnetCDF + run: | + echo "PNETCDF=/opt/pnetcdf" >> $GITHUB_ENV + + ########################################################################################## + # Clone and build the MPAS baseline branch. + ########################################################################################## + - name: Get SHA -- baseline branch + id: baseline-sha + run: | + echo "sha=$(git ls-remote https://github.com/ufs-community/MPAS-Model.git noaa/develop | cut -f1)" >> $GITHUB_OUTPUT + + - name: Checkout MPAS -- baseline branch + run: | + cd ${runner_ROOT} + git clone --recursive --branch noaa/develop https://github.com/ufs-community/MPAS-Model.git MPAS-Model-BL + + - name: Cache MPAS build -- baseline branch + id: cache-bl + uses: actions/cache@v3 + with: + path: ${{env.SRCDIR_BL}}/atmosphere_model + key: mpas-build-bl-${{runner.os}}-${{matrix.image}}-${{matrix.f-compiler}}-${{matrix.build-type}}-${{steps.baseline-sha.outputs.sha}} + + - name: Download WRF physics tables -- baseline branch + run: | + cd ${SRCDIR_BL}/src/core_atmosphere/physics + ./checkout_data_files.sh + + - name: Build MPAS -- baseline branch + if: steps.cache-bl.outputs.cache-hit != 'true' + run: | + cd ${SRCDIR_BL} + make ${{matrix.bld_target}} CORE=atmosphere ${{matrix.build-type == 'Debug' && 'DEBUG=true' || ''}} + + ########################################################################################## + # Clone and build the MPAS feature branch. + ########################################################################################## + + # actions/checkout@v3 is a standard GitHub Actions action that clones the repository that + # triggered the workflow into the runner's workspace. Specifically it: + # + # 1) Clones the feature branch (the one that was pushed or opened as a PR) into ${SRCDIR_FT}. + # 2) Checks out the specific commit that triggered the workflow. + # + - name: Checkout MPAS -- feature branch + uses: actions/checkout@v3 + + - name: Initialize submodules -- feature branch + run: | + git config --global --add safe.directory ${SRCDIR_FT} + cd ${SRCDIR_FT} + git submodule update --init --recursive + + - name: Cache MPAS build -- feature branch + id: cache-feature + uses: actions/cache@v3 + with: + path: ${{env.SRCDIR_FT}}/atmosphere_model + key: mpas-build-${{runner.os}}-${{matrix.image}}-${{matrix.f-compiler}}-${{matrix.build-type}}-${{hashFiles('Makefile', 'src/**')}} + + - name: Download WRF physics tables -- feature branch + run: | + cd ${SRCDIR_FT}/src/core_atmosphere/physics + ./checkout_data_files.sh + + - name: Build MPAS -- feature branch + if: steps.cache-feature.outputs.cache-hit != 'true' + run: | + cd ${SRCDIR_FT} + make ${{matrix.bld_target}} CORE=atmosphere ${{matrix.build-type == 'Debug' && 'DEBUG=true' || ''}} + + ########################################################################################## + # Download any data/files needed for MPAS runs. + # This is done for both branches since the files downloaded by the get_data.sh script may + # not be the same for the two branches. + ########################################################################################## + - name: Download MPAS static/IC/LBC files and UFS physics tables -- baseline branch + run: | + ${SRCDIR_BL}/${ufscmnty_data}/get_data.sh + + - name: Download MPAS static/IC/LBC files and UFS physics tables -- feature branch + run: | + ${SRCDIR_FT}/${ufscmnty_data}/get_data.sh + + ########################################################################################## + # Configure MPAS baseline branch run directory. + ########################################################################################## + - name: Create and populate run directory -- baseline branch runs + run: | + rundir=${RUNDIR_BL} + srcdir=${SRCDIR_BL} + # Create and cd into run directory. + mkdir -p ${rundir} && cd ${rundir} + # Copy MPAS configuration files (namelist, stream) into run directory. + # + # Since the baseline branch does not yet have a "/some/long/path.stoch" directory from + # which to copy the stochastic physics configuration files, the following command for + # copying the MPAS configuration files to the run directory will not work. Thus, + # comment this out for now and uncomment only after the stochastic physics PR is + # merged. + #cp ${srcdir}/${ufscmnty_cases}/ufscommunity.${{matrix.physics}}.${{matrix.ic_source}}.${{matrix.season}}.stoch/* . + # + # Instead, copy the MPAS configuration files from the corresponding directory for the + # feature branch. Also, since the namelist file that the baseline branch cannot + # contain the stochastic physics namelist (called "nam_stochy", we specifically copy + # in a version of the namelist file without that namelist. + cp ${SRCDIR_FT}/${ufscmnty_cases}/ufscommunity.${{matrix.physics}}.${{matrix.ic_source}}.${{matrix.season}}.stoch/* . + # The following command takes the namelist.atmosphere file copied into the run directory + # above (and which contains the nam_stochy namelist) and removes that namelist from it. + sed -i '/^[[:space:]]*&nam_stochy/,/^[[:space:]]*\//d' namelist.atmosphere + # + # Create symlinks in the run directory. + # + # Links to WRF physics tables and other files that are downloaded during the call to + # checkout_data_files.sh. + ln -sf ${srcdir}/${wrf_phys_tables}/{*.TBL,*.DBL,*DATA} . + # Links to UFS physics tables that are under version control, i.e. part of the MPAS-Model + # repo or submodules. These are not created in the build step or some other way. They + # are present once the repository is cloned (recursively). + ln -sf ${srcdir}/src/core_atmosphere/physics/physics_noahmp/parameters/NoahmpTable.TBL . + ln -sf ${srcdir}/src/core_atmosphere/physics/physics_noaa/TEMPO/tables/ccn_activate.bin . + # Links to the UFS physics tables downloaded by the get_data.sh script. + ln -sf ${srcdir}/${ufs_phys_tables}/tempo/* . + ln -sf ${srcdir}/${ufs_phys_tables}/ugw/* . + # Links to the MPAS initial and boundary condition files and other static files. + ln -sf ${srcdir}/${mpas_ics}/ufscommunity.${{matrix.ic_source}}.${{matrix.season}}/mpas.init.nc . + ln -sf ${srcdir}/${mpas_ics}/ufscommunity.${{matrix.ic_source}}.${{matrix.season}}/mpas.lbc.nc . + ln -sf ${srcdir}/${mpas_ics}/ufscommunity.${{matrix.ic_source}}.${{matrix.season}}/mpas.ugwp_oro_data.nc . + # Link to the MPAS executable. + ln -sf ${srcdir}/atmosphere_model . + + - name: Link surface files to run directory -- baseline branch runs + if: matrix.ic_source == 'gfs' + run: | + cd ${RUNDIR_BL} + ln -sf ${SRCDIR_BL}/${mpas_ics}/ufscommunity.${{matrix.ic_source}}.${{matrix.season}}/mpas.sfc_update.nc . + + ########################################################################################## + # Configure MPAS feature branch run directory. + ########################################################################################## + - name: Create and populate run directory -- feature branch runs + run: | + rundir=${RUNDIR_FT} + srcdir=${SRCDIR_FT} + # Create and cd into run directory. + mkdir -p ${rundir} && cd ${rundir} + # Copy MPAS configuration files (namelist, stream) into run directory. + cp ${srcdir}/${ufscmnty_cases}/ufscommunity.${{matrix.physics}}.${{matrix.ic_source}}.${{matrix.season}}.stoch/* . + # + # Create symlinks in the run directory. + # + # Links to WRF physics tables and other files that are downloaded during the call to + # checkout_data_files.sh. + ln -sf ${srcdir}/${wrf_phys_tables}/{*.TBL,*.DBL,*DATA} . + # Links to UFS physics tables that are under version control, i.e. part of the MPAS-Model + # repo or submodules. These are not created in the build step or some other way. They + # are present once the repository is cloned (recursively). + ln -sf ${srcdir}/src/core_atmosphere/physics/physics_noahmp/parameters/NoahmpTable.TBL . + ln -sf ${srcdir}/src/core_atmosphere/physics/physics_noaa/TEMPO/tables/ccn_activate.bin . + # Links to the UFS physics tables downloaded by the get_data.sh script. + ln -sf ${srcdir}/${ufs_phys_tables}/tempo/* . + ln -sf ${srcdir}/${ufs_phys_tables}/ugw/* . + # Links to the MPAS initial and boundary condition files and other static files. + ln -sf ${srcdir}/${mpas_ics}/ufscommunity.${{matrix.ic_source}}.${{matrix.season}}/mpas.init.nc . + ln -sf ${srcdir}/${mpas_ics}/ufscommunity.${{matrix.ic_source}}.${{matrix.season}}/mpas.lbc.nc . + ln -sf ${srcdir}/${mpas_ics}/ufscommunity.${{matrix.ic_source}}.${{matrix.season}}/mpas.ugwp_oro_data.nc . + # Link to the MPAS executable. + ln -sf ${srcdir}/atmosphere_model . + + - name: Link surface files to run directory -- feature branch runs + if: matrix.ic_source == 'gfs' + run: | + cd ${RUNDIR_FT} + ln -sf ${SRCDIR_FT}/${mpas_ics}/ufscommunity.${{matrix.ic_source}}.${{matrix.season}}/mpas.sfc_update.nc . + + ########################################################################################## + # Run MPAS with both baseline and feature branches. + ########################################################################################## + - name: Run MPAS -- baseline branch + run: | + cd ${RUNDIR_BL} + pwd && ls -l + mpiexec --allow-run-as-root -np 1 ./atmosphere_model || true + pwd && ls -l + + - name: Run MPAS -- feature branch + run: | + cd ${RUNDIR_FT} + pwd && ls -l + mpiexec --allow-run-as-root -np 1 ./atmosphere_model || true + pwd && ls -l + + ########################################################################################## + # Compare feature branch to baseline branch. + ########################################################################################## + +# # Note that the comparison script used below (cmp_rt2bl.py) is the one in the feature +# # branch. Thus, the environment file used in the miniconda setup should be the one in +# # that branch (SRCDIR_FT). Since the miniconda step prepends ${GITHUB_WORKSPACE} to +# # whatever value is provided to environment-file: below and since ${SRCDIR_FT} is set +# # to ${GITHUB_WORKSPACE}, the environment file used below is in fact the one in the +# # feature branch. +# - name: Setup miniconda +# uses: conda-incubator/setup-miniconda@v3 +# with: +# python-version: ${{env.py-version}} +# auto-update-conda: true +# auto-activate: false +# activate-environment: mpas-ci-env +# environment-file: ${{env.ufs_community}}/environment.yml + + - name: Cache conda environment + id: cache-conda + uses: actions/cache@v3 + with: + path: ${{ env.CONDA }}/envs + key: conda-${{ runner.os }}-${{ hashFiles(format('{0}/environment.yml', env.ufs_community)) }} + + - name: Setup miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{env.py-version}} + auto-update-conda: true + auto-activate: false + activate-environment: mpas-ci-env + + - name: Create conda environment + if: steps.cache-conda.outputs.cache-hit != 'true' + run: conda env update --file ${SRCDIR_FT}/${{env.ufs_community}}/environment.yml --name mpas-ci-env + + - name: Run comparison script + run: | + cd ${SRCDIR_FT}/${ufs_community} + # Note that if the directory for the baseline run ${RUNDIR_BL} doesn't exist, the + # comparison script cmp_rt2bl.py will still report a success. Not clear if this + # is by design. + ./cmp_rt2bl.py --dir_rt ${RUNDIR_FT} --dir_bl ${RUNDIR_BL} + + ########################################################################################## + # Save MPAS output and log files as GitHub Artifact. + ########################################################################################## + - name: Create GitHub artifact + run: | + datadir="${ARTIFACT_DIR}/data_bl" + mkdir -p ${datadir} + cd ${RUNDIR_BL} + cp ./* ${datadir} + # + datadir="${ARTIFACT_DIR}/data_ft" + mkdir -p ${datadir} + cd ${RUNDIR_FT} + cp ./* ${datadir} + + - name: Upload MPAS runs as GitHub artifact + uses: actions/upload-artifact@v4 + with: + name: mpas-baselines-${{env.artifact}} + path: ${{env.ARTIFACT_DIR}} diff --git a/.gitmodules b/.gitmodules index bdf041fe6b..f19f95960e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,3 +22,6 @@ [submodule "src/core_atmosphere/physics/physics_mmm"] path = src/core_atmosphere/physics/physics_mmm url = https://github.com/NCAR/MMM-physics.git +[submodule "src/core_atmosphere/stochastic_physics"] + path = src/core_atmosphere/stochastic_physics + url = https://github.com/dtcenter/stochastic_physics.git diff --git a/CMakeLists.txt b/CMakeLists.txt index e7e010394d..864675f5eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -122,6 +122,13 @@ foreach(_core IN LISTS MPAS_CORES) add_subdirectory(src/core_${_core}) # Target: MPAS::core:: endforeach() +# When both atmosphere and init_atmosphere cores are built, ifort transitively +# resolves atm_core_interface.mod while compiling mpas.F for mpas_init_atmosphere. +# Ensure core_atmosphere (which produces atm_core_interface.mod) is built first. +if("atmosphere" IN_LIST MPAS_CORES AND "init_atmosphere" IN_LIST MPAS_CORES) + add_dependencies(mpas_init_atmosphere core_atmosphere) +endif() + ### Package config include(CMakePackageConfigHelpers) diff --git a/Makefile b/Makefile index 9b12a88df6..7124e96fdc 100644 --- a/Makefile +++ b/Makefile @@ -481,6 +481,35 @@ intel-mpi: # BUILDTARGET Intel compiler suite with Intel MPI library "DEBUG = $(DEBUG)" \ "USE_PAPI = $(USE_PAPI)" \ "OPENMP = $(OPENMP)" \ + "LAPACK_LIBS = -lmkl_intel_lp64 -lmkl_core -lmkl_sequential" \ + "CPPFLAGS = $(MODEL_FORMULATION) -D_MPI" ) + +intel-mpi-gaeac6: # BUILDTARGET for gaea C6 + ( $(MAKE) all \ + "FC_PARALLEL = ftn" \ + "CC_PARALLEL = cc" \ + "CXX_PARALLEL = mpiicpc" \ + "FC_SERIAL = ifort" \ + "CC_SERIAL = cc" \ + "CXX_SERIAL = icpc" \ + "FFLAGS_PROMOTION = -real-size 64" \ + "FFLAGS_OPT = -O3 -convert big_endian -free -align array64byte" \ + "CFLAGS_OPT = -O3" \ + "CXXFLAGS_OPT = -O3" \ + "LDFLAGS_OPT = -O3" \ + "FFLAGS_DEBUG = -g -convert big_endian -free -CU -CB -check all -fpe0 -traceback" \ + "CFLAGS_DEBUG = -g -traceback" \ + "CXXFLAGS_DEBUG = -g -traceback" \ + "LDFLAGS_DEBUG = -g -fpe0 -traceback" \ + "FFLAGS_OMP = -qopenmp" \ + "CFLAGS_OMP = -qopenmp" \ + "PICFLAG = -fpic" \ + "BUILD_TARGET = $(@)" \ + "CORE = $(CORE)" \ + "DEBUG = $(DEBUG)" \ + "USE_PAPI = $(USE_PAPI)" \ + "OPENMP = $(OPENMP)" \ + "LAPACK_LIBS = -lmkl_intel_lp64 -lmkl_core -lmkl_sequential" \ "CPPFLAGS = $(MODEL_FORMULATION) -D_MPI" ) intel-llvm-mpi: # BUILDTARGET Intel LLVM compiler suite with Intel MPI library @@ -565,6 +594,7 @@ gfortran: # BUILDTARGET GNU Fortran, C, and C++ compilers "USE_PAPI = $(USE_PAPI)" \ "OPENMP = $(OPENMP)" \ "OPENACC = $(OPENACC)" \ + "LAPACK_LIBS = -llapack -lblas" \ "CPPFLAGS = $(MODEL_FORMULATION) -D_MPI" ) gfortran-clang: # BUILDTARGET GNU Fortran compiler with LLVM clang/clang++ compilers diff --git a/cmake/Functions/MPAS_Functions.cmake b/cmake/Functions/MPAS_Functions.cmake index fe76556225..0a802c67eb 100644 --- a/cmake/Functions/MPAS_Functions.cmake +++ b/cmake/Functions/MPAS_Functions.cmake @@ -214,24 +214,24 @@ function(mpas_core_target) set(CPP_EXTRA_FLAGS ${CPP_EXTRA_FLAGS} -DDO_PHYSICS) endif() -add_custom_command(OUTPUT Registry_processed.xml - COMMAND ${CPP_EXECUTABLE} -E -P ${CPP_EXTRA_FLAGS} ${CMAKE_CURRENT_SOURCE_DIR}/Registry.xml > Registry_processed.xml + add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/Registry_processed.xml + COMMAND ${CPP_EXECUTABLE} -E -P ${CPP_EXTRA_FLAGS} ${CMAKE_CURRENT_SOURCE_DIR}/Registry.xml > ${CMAKE_CURRENT_BINARY_DIR}/Registry_processed.xml COMMENT "CORE ${ARG_CORE}: Pre-Process Registry" - DEPENDS Registry.xml) + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Registry.xml) add_custom_command(OUTPUT ${ARG_INCLUDES} - COMMAND mpas_parse_${ARG_CORE} Registry_processed.xml ${CPP_EXTRA_FLAGS} + COMMAND mpas_parse_${ARG_CORE} ${CMAKE_CURRENT_BINARY_DIR}/Registry_processed.xml ${CPP_EXTRA_FLAGS} COMMENT "CORE ${ARG_CORE}: Parse Registry" - DEPENDS mpas_parse_${ARG_CORE} Registry_processed.xml) + DEPENDS mpas_parse_${ARG_CORE} ${CMAKE_CURRENT_BINARY_DIR}/Registry_processed.xml) add_custom_command(OUTPUT namelist.${ARG_CORE} WORKING_DIRECTORY ${CORE_DATADIR} COMMAND mpas_namelist_gen ${CMAKE_CURRENT_BINARY_DIR}/Registry_processed.xml namelist.${ARG_CORE} in_defaults=true COMMENT "CORE ${ARG_CORE}: Generate Namelist" - DEPENDS mpas_namelist_gen Registry_processed.xml) + DEPENDS mpas_namelist_gen ${CMAKE_CURRENT_BINARY_DIR}/Registry_processed.xml) add_custom_command(OUTPUT streams.${ARG_CORE} WORKING_DIRECTORY ${CORE_DATADIR} COMMAND mpas_streams_gen ${CMAKE_CURRENT_BINARY_DIR}/Registry_processed.xml streams.${ARG_CORE} stream_list.${ARG_CORE}. listed COMMENT "CORE ${ARG_CORE}: Generate Streams" - DEPENDS mpas_streams_gen Registry_processed.xml) + DEPENDS mpas_streams_gen ${CMAKE_CURRENT_BINARY_DIR}/Registry_processed.xml) add_custom_target(gen_${ARG_CORE} DEPENDS ${ARG_INCLUDES} namelist.${ARG_CORE} streams.${ARG_CORE}) add_dependencies(${ARG_TARGET} gen_${ARG_CORE}) diff --git a/modulefiles/mpas/derecho.intel.lua b/modulefiles/mpas/derecho.intel.lua index 228346b66f..d4c6d1b3e3 100644 --- a/modulefiles/mpas/derecho.intel.lua +++ b/modulefiles/mpas/derecho.intel.lua @@ -7,15 +7,20 @@ prepend_path("MODULEPATH", '/glade/work/epicufsrt/contrib/spack-stack/derecho/sp load("stack-oneapi/2024.2.1") load("stack-cray-mpich/8.1.29") +load("intel-oneapi-mkl/2024.2.2") load("cmake/3.27.9") load("parallel-netcdf/1.12.3") load("parallelio/2.6.2") if mode() == "load" then setenv("PNETCDF", os.getenv("parallel_netcdf_ROOT")) + setenv("NetCDF_C_ROOT", os.getenv("netcdf_c_ROOT")) + setenv("NetCDF_FORTRAN_ROOT", os.getenv("netcdf_fortran_ROOT")) end if mode() == "unload" then unsetenv("PNETCDF") + unsetenv("NetCDF_C_ROOT") + unsetenv("NetCDF_FORTRAN_ROOT") end setenv("CMAKE_C_COMPILER", "mpicc") diff --git a/modulefiles/mpas/gaeac6.intel.lua b/modulefiles/mpas/gaeac6.intel.lua index dc187dc252..4a92ab937e 100644 --- a/modulefiles/mpas/gaeac6.intel.lua +++ b/modulefiles/mpas/gaeac6.intel.lua @@ -7,15 +7,20 @@ prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.9.3/env load("stack-oneapi/2024.2.1") load("stack-cray-mpich/8.1.32") +load("intel-oneapi-mkl/2023.2.0") load("cmake/3.27.9") load("parallel-netcdf/1.12.3") load("parallelio/2.6.2") if mode() == "load" then setenv("PNETCDF", os.getenv("parallel_netcdf_ROOT")) + setenv("NetCDF_C_ROOT", os.getenv("netcdf_c_ROOT")) + setenv("NetCDF_FORTRAN_ROOT", os.getenv("netcdf_fortran_ROOT")) end if mode() == "unload" then unsetenv("PNETCDF") + unsetenv("NetCDF_C_ROOT") + unsetenv("NetCDF_FORTRAN_ROOT") end setenv("CMAKE_C_COMPILER", "mpicc") diff --git a/modulefiles/mpas/hera.intel.lua b/modulefiles/mpas/hera.intel.lua index fd2aa48302..e0eaa3f180 100644 --- a/modulefiles/mpas/hera.intel.lua +++ b/modulefiles/mpas/hera.intel.lua @@ -7,15 +7,20 @@ prepend_path("MODULEPATH","/contrib/spack-stack/spack-stack-1.9.3/envs/ue-oneapi load("stack-oneapi/2024.2.1") load("stack-intel-oneapi-mpi/2021.13") +load("mkl/2024.2.1") load("cmake/3.27.9") load("parallel-netcdf/1.12.3") load("parallelio/2.6.2") if mode() == "load" then setenv("PNETCDF", os.getenv("parallel_netcdf_ROOT")) + setenv("NetCDF_C_ROOT", os.getenv("netcdf_c_ROOT")) + setenv("NetCDF_FORTRAN_ROOT", os.getenv("netcdf_fortran_ROOT")) end if mode() == "unload" then unsetenv("PNETCDF") + unsetenv("NetCDF_C_ROOT") + unsetenv("NetCDF_FORTRAN_ROOT") end setenv("CMAKE_C_COMPILER", "mpiicc") diff --git a/modulefiles/mpas/orion.intel.lua b/modulefiles/mpas/orion.intel.lua index 89c14e7a1e..74ef0f75f5 100644 --- a/modulefiles/mpas/orion.intel.lua +++ b/modulefiles/mpas/orion.intel.lua @@ -7,15 +7,20 @@ prepend_path("MODULEPATH", "/apps/contrib/spack-stack/spack-stack-1.9.3/envs/ue- load("stack-oneapi/2024.2.1") load("stack-intel-oneapi-mpi/2021.13") +load("intel-oneapi-mkl/2024.2.1") load("cmake/3.27.9") load("parallel-netcdf/1.12.3") load("parallelio/2.6.2") if mode() == "load" then setenv("PNETCDF", os.getenv("parallel_netcdf_ROOT")) + setenv("NetCDF_C_ROOT", os.getenv("netcdf_c_ROOT")) + setenv("NetCDF_FORTRAN_ROOT", os.getenv("netcdf_fortran_ROOT")) end if mode() == "unload" then unsetenv("PNETCDF") + unsetenv("NetCDF_C_ROOT") + unsetenv("NetCDF_FORTRAN_ROOT") end setenv("CMAKE_C_COMPILER", "mpiicc") diff --git a/modulefiles/mpas/ursa.intel.lua b/modulefiles/mpas/ursa.intel.lua index 2b2dc15cf1..f701e51442 100644 --- a/modulefiles/mpas/ursa.intel.lua +++ b/modulefiles/mpas/ursa.intel.lua @@ -7,7 +7,7 @@ prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.9.3/envs/ue-oneap load("stack-oneapi/2024.2.1") load("stack-intel-oneapi-mpi/2021.13") - +load("intel-oneapi-mkl/2024.2.1") load("cmake/3.27.9") load("parallel-netcdf/1.12.3") load("parallelio/2.6.2") @@ -17,9 +17,12 @@ setenv("CMAKE_CXX_COMPILER", "mpiicpc") setenv("CMAKE_Fortran_COMPILER", "mpiifort") if mode() == "load" then - --setenv("PNETCDF", os.getenv("PARALLEL_NETCDF_ROOT")) setenv("PNETCDF", os.getenv("parallel_netcdf_ROOT")) + setenv("NetCDF_C_ROOT", os.getenv("netcdf_c_ROOT")) + setenv("NetCDF_FORTRAN_ROOT", os.getenv("netcdf_fortran_ROOT")) end if mode() == "unload" then unsetenv("PNETCDF") + unsetenv("NetCDF_C_ROOT") + unsetenv("NetCDF_FORTRAN_ROOT") end diff --git a/src/Makefile b/src/Makefile index b9c037c8cc..cd133bd373 100644 --- a/src/Makefile +++ b/src/Makefile @@ -9,7 +9,7 @@ else all: mpas mpas: $(AUTOCLEAN_DEPS) externals frame ops dycore drver - $(LINKER) $(LDFLAGS) -o $(EXE_NAME) driver/*.o -L. -ldycore -lops -lframework $(LIBS) -I./external/esmf_time_f90 -L./external/esmf_time_f90 -lesmf_time + $(LINKER) $(LDFLAGS) -o $(EXE_NAME) driver/*.o -L. -ldycore -lops -lframework $(LIBS) -I./external/esmf_time_f91 -L./external/esmf_time_f90 -lesmf_time $(LAPACK_LIBS) externals: $(AUTOCLEAN_DEPS) ( cd external; $(MAKE) FC="$(FC)" SFC="$(SFC)" CC="$(CC)" SCC="$(SCC)" FFLAGS="$(FFLAGS)" CFLAGS="$(CFLAGS)" CPP="$(CPP)" NETCDF="$(NETCDF)" CORE="$(CORE)" all ) diff --git a/src/core_atmosphere/CMakeLists.txt b/src/core_atmosphere/CMakeLists.txt index c8de66f3c7..e7151c5f41 100644 --- a/src/core_atmosphere/CMakeLists.txt +++ b/src/core_atmosphere/CMakeLists.txt @@ -140,6 +140,39 @@ list(TRANSFORM ATMOSPHERE_CORE_PHYSICS_SMOKE_SOURCES PREPEND physics/physics_noa set(ATMOSPHERE_CORE_PHYSICS_MMM_DIR ${CMAKE_CURRENT_SOURCE_DIR}/physics/physics_mmm) +# build stochastic_physics as a library +set(ATMOSPHERE_CORE_STOCHASTIC_PHYSICS_SOURCES + kinddef.F90 + mpi_wrapper.F90 + plumes.F90 + mersenne_twister.F90 + random_numbers.F90 + stochy_namelist_def.F90 + compns_stochy.F90 + stochy_internal_state_mod.F90 + spectral_transforms.F90 + stochastic_physics_m.F90 + stochy_patterngenerator.F90 + stochy_data_mod.F90 + stochy_nml_rec.F90 + get_stochy_pattern.F90 + lndp_apply_perts.F90 + mpas_stochastic_physics.F +) +list(TRANSFORM ATMOSPHERE_CORE_STOCHASTIC_PHYSICS_SOURCES PREPEND stochastic_physics/) +add_library(stochastic_physics ${ATMOSPHERE_CORE_STOCHASTIC_PHYSICS_SOURCES}) +mpas_fortran_target(stochastic_physics) +target_compile_definitions(stochastic_physics PRIVATE INTERNAL_FILE_NML MPAS) +if(NOT MPAS_DOUBLE_PRECISION) + target_compile_definitions(stochastic_physics PRIVATE CCPP_32BIT) +endif() +find_package(BLAS REQUIRED) +find_package(NetCDF REQUIRED COMPONENTS Fortran C) +target_link_libraries(stochastic_physics PUBLIC MPI::MPI_Fortran framework ${BLAS_LIBRARIES} NetCDF::NetCDF_Fortran NetCDF::NetCDF_C) +install(TARGETS stochastic_physics EXPORT ${PROJECT_NAME}ExportsCore + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) + if(NOT EXISTS ${ATMOSPHERE_CORE_PHYSICS_MMM_DIR}) set(PHYSICS_MMM_REPO_URL "https://github.com/NCAR/MMM-physics") execute_process(COMMAND git clone ${PHYSICS_MMM_REPO_URL} ${ATMOSPHERE_CORE_PHYSICS_MMM_DIR} @@ -532,6 +565,7 @@ if (${DO_PHYSICS}) list(APPEND CORE_ATMOSPHERE_COMPILE_DEFINITIONS DO_PHYSICS) endif () target_compile_definitions(core_atmosphere PRIVATE ${CORE_ATMOSPHERE_COMPILE_DEFINITIONS}) +target_link_libraries(core_atmosphere PUBLIC stochastic_physics) set_MPAS_DEBUG_flag(core_atmosphere) mpas_core_target(CORE atmosphere TARGET core_atmosphere INCLUDES ${ATMOSPHERE_CORE_INCLUDES}) diff --git a/src/core_atmosphere/Makefile b/src/core_atmosphere/Makefile index 966027bc77..92bb34cfa3 100644 --- a/src/core_atmosphere/Makefile +++ b/src/core_atmosphere/Makefile @@ -28,6 +28,7 @@ core_reg: core_input_gen: if [ ! -e default_inputs ]; then mkdir default_inputs; fi + rm -f default_inputs/streams.atmosphere default_inputs/stream_list.atmosphere.* ( cd default_inputs; $(NL_GEN) ../Registry_processed.xml namelist.atmosphere in_defaults=true ) ( cd default_inputs; $(ST_GEN) ../Registry_processed.xml streams.atmosphere stream_list.atmosphere. listed in_defaults=true) @@ -37,12 +38,21 @@ gen_includes: core_reg post_build: if [ ! -e $(ROOT_DIR)/default_inputs ]; then mkdir $(ROOT_DIR)/default_inputs; fi + rm -f $(ROOT_DIR)/default_inputs/streams.$(CORE) $(ROOT_DIR)/default_inputs/stream_list.$(CORE).* cp default_inputs/* $(ROOT_DIR)/default_inputs/. - ( cd $(ROOT_DIR)/default_inputs; for FILE in `ls -1`; do if [ ! -e ../$$FILE ]; then cp $$FILE ../.; fi; done ) + ( cd $(ROOT_DIR)/default_inputs; for FILE in `ls -1`; do \ + if [ -e ../$$FILE ]; then \ + N=1; \ + while [ -e "../$$FILE.old$$(printf '%03d' $$N)" ]; do N=$$((N+1)); done; \ + mv "../$$FILE" "../$$FILE.old$$(printf '%03d' $$N)"; \ + fi; \ + cp $$FILE ../.; \ + done ) physcore: mpas_atm_dimensions.o ( cd physics; $(MAKE) all ) - ( mkdir libphys; cd libphys; ar -x ../physics/libphys.a ) + ( cd stochastic_physics; $(MAKE) -f Makefile all ) + ( mkdir libphys; cd libphys; ar -x ../physics/libphys.a; ar -x ../stochastic_physics/libstochphys.a ) ( cd ../..; ln -sf ./src/core_atmosphere/physics/physics_wrf/files/*TBL .) ( cd ../..; ln -sf ./src/core_atmosphere/physics/physics_wrf/files/*DATA* .) ( cd ../..; ln -sf ./src/core_atmosphere/physics/physics_noahmp/parameters/*TBL .) @@ -68,6 +78,12 @@ mpas_atm_dimensions.o: clean: ( cd physics; $(MAKE) clean ) ( cd dynamics; $(MAKE) clean ) + @# The stochastic_physics directory contains both a makefile (lower case + @# "m") and a Makefile (upper case "M"). The former is used for FV3 and + @# the latter for MPAS. The default for "make" is to use "makefile", but + @# here, we want to use the one for MPAS (with an upper case "M"). Indicate + @# that using the -f flag. + ( cd stochastic_physics; $(MAKE) -f Makefile clean ) ( cd diagnostics; $(MAKE) clean ) ( cd utils; $(MAKE) clean ) ( cd ../..; rm -f *TBL ) @@ -84,7 +100,7 @@ clean: $(RM) $@ $*.mod ifeq "$(GEN_F90)" "true" $(CPP) $(CPPFLAGS) $(PHYSICS) $(CPPINCLUDES) -I./inc $< > $*.f90 - $(FC) $(FFLAGS) -c $*.f90 $(FCINCLUDES) -I../framework -I../operators -I./physics -I./dynamics -I./diagnostics -I./physics/physics_wrf -I./physics/physics_mmm -I./physics/physics_noaa/UGWP -I../external/esmf_time_f90 + $(FC) $(FFLAGS) -c $*.f90 $(FCINCLUDES) -I../framework -I../operators -I./physics -I./dynamics -I./diagnostics -I./physics/physics_wrf -I./physics/physics_mmm -I./physics/physics_noaa/UGWP -I./stochastic_physics -I../external/esmf_time_f90 else - $(FC) $(CPPFLAGS) $(PHYSICS) $(FFLAGS) -c $*.F $(CPPINCLUDES) $(FCINCLUDES) -I./inc -I../framework -I../operators -I./physics -I./dynamics -I./diagnostics -I./physics/physics_wrf -I./physics/physics_mmm -I./physics/physics_noaa/UGWP -I../external/esmf_time_f90 + $(FC) $(CPPFLAGS) $(PHYSICS) $(FFLAGS) -c $*.F $(CPPINCLUDES) $(FCINCLUDES) -I./inc -I../framework -I../operators -I./physics -I./dynamics -I./diagnostics -I./physics/physics_wrf -I./physics/physics_mmm -I./physics/physics_noaa/UGWP -I./stochastic_physics -I../external/esmf_time_f90 endif diff --git a/src/core_atmosphere/Registry.xml b/src/core_atmosphere/Registry.xml index 4851e1cae7..985cabd5dc 100644 --- a/src/core_atmosphere/Registry.xml +++ b/src/core_atmosphere/Registry.xml @@ -32,6 +32,10 @@ description="The number of atmospheric layers"/> + + #ifdef DO_PHYSICS + + @@ -4162,6 +4168,11 @@ description="Total cell-centered meridional wind tendency from physics" persistence="scratch" /> + + + #ifdef DO_PHYSICS @@ -4672,5 +4683,6 @@ #include "physics/Registry_noahmp.xml" #include "physics/Registry_tempo.xml" #include "physics/registry.chemistry.xml" +#include "stochastic_physics/Registry_stoch_physics.xml" #endif diff --git a/src/core_atmosphere/dynamics/Makefile b/src/core_atmosphere/dynamics/Makefile index 6892633c68..62d595c90a 100644 --- a/src/core_atmosphere/dynamics/Makefile +++ b/src/core_atmosphere/dynamics/Makefile @@ -22,5 +22,5 @@ ifeq "$(GEN_F90)" "true" $(CPP) $(CPPFLAGS) $(PHYSICS) $(CPPINCLUDES) $< > $*.f90 $(FC) $(FFLAGS) -c $*.f90 $(FCINCLUDES) -I.. -I../../framework -I../../operators -I../physics -I../physics/physics_wrf -I../physics/physics_mmm -I../../external/esmf_time_f90 else - $(FC) $(CPPFLAGS) $(PHYSICS) $(FFLAGS) -c $*.F $(CPPINCLUDES) $(FCINCLUDES) -I.. -I../../framework -I../../operators -I../physics -I../physics/physics_wrf -I../physics/physics_mmm -I../../external/esmf_time_f90 + $(FC) $(CPPFLAGS) $(PHYSICS) $(FFLAGS) -c $*.F $(CPPINCLUDES) $(FCINCLUDES) -I.. -I../../framework -I../../operators -I../physics -I../physics/physics_wrf -I../physics/physics_mmm -I../stochastic_physics -I../../external/esmf_time_f90 endif diff --git a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F index 0601020cde..d49d58d1ea 100644 --- a/src/core_atmosphere/dynamics/mpas_atm_time_integration.F +++ b/src/core_atmosphere/dynamics/mpas_atm_time_integration.F @@ -31,6 +31,7 @@ module atm_time_integration use mpas_atmphys_driver_microphysics use mpas_atmphys_todynamics use mpas_atmphys_utilities + use mpas_stochastic_physics, only : stochastic_physics_pattern_apply, dosppt #endif use mpas_atm_boundaries, only : nSpecZone, nRelaxZone, nBdyZone, mpas_atm_get_bdy_state, mpas_atm_get_bdy_tend ! regional_MPAS addition @@ -880,6 +881,8 @@ subroutine atm_srk3(domain, dt, itimestep, exchange_halo_group) #endif real (kind=RKIND) :: time_dyn_step + character(len=32) :: tend_names(4) + integer :: ierr logical, parameter :: debug = .false. @@ -1077,11 +1080,28 @@ subroutine atm_srk3(domain, dt, itimestep, exchange_halo_group) call mpas_timer_start('physics_get_tend') rk_step = 1 dynamics_substep = 1 + +! apply random perturbation pattern to the tendency + tend_names(1) = "rucuten" + tend_names(2) = "rvcuten" + tend_names(3) = "rublten" + tend_names(4) = "rvblten" + if (dosppt(domain)) then + call stochastic_physics_pattern_apply(domain, 4, tend_names, ierr) + endif + call physics_get_tend( block, mesh, state, diag, tend, tend_physics, & block % configs, rk_step, dynamics_substep, & tend_ru_physics, tend_rtheta_physics, tend_rho_physics, & exchange_halo_group ) call mpas_timer_stop('physics_get_tend') + +! apply random perturbation pattern to the tendency + tend_names(1) = "tend_rtheta_physics" +! tend_names(2) = "tend_rho_physics" + if (dosppt(domain)) then + call stochastic_physics_pattern_apply(domain, 1, tend_names, ierr) + endif #else #ifndef MPAS_CAM_DYCORE ! diff --git a/src/core_atmosphere/mpas_atm_core.F b/src/core_atmosphere/mpas_atm_core.F index e10171e8c8..c0ac5c6cc8 100644 --- a/src/core_atmosphere/mpas_atm_core.F +++ b/src/core_atmosphere/mpas_atm_core.F @@ -48,6 +48,7 @@ function atm_core_init(domain, startTimeStamp) result(ierr) use mpas_attlist, only : mpas_modify_att use mpas_string_utils, only : mpas_string_replace use mpas_atm_halos, only: atm_build_halo_groups, exchange_halo_group + use mpas_stochastic_physics, only : stochastic_physics_pattern_init, dosppt implicit none @@ -301,6 +302,11 @@ function atm_core_init(domain, startTimeStamp) result(ierr) ! Prepare the dynamics for integration ! call mpas_atm_dynamics_init(domain) + ! + ! init stochastic pattern generation + if (dosppt(domain)) then + call stochastic_physics_pattern_init (domain, ierr) + endif end function atm_core_init @@ -1014,6 +1020,7 @@ subroutine atm_do_timestep(domain, dt, itimestep) use mpas_atmphys_update #endif use mpas_atm_halos, only: exchange_halo_group + use mpas_stochastic_physics, only : stochastic_physics_pattern_adv, dosppt implicit none @@ -1046,6 +1053,9 @@ subroutine atm_do_timestep(domain, dt, itimestep) endif #endif + if (dosppt(domain)) then + call stochastic_physics_pattern_adv(domain, itimestep, ierr) + endif call atm_timestep(domain, dt, currTime, itimestep, exchange_halo_group) end subroutine atm_do_timestep diff --git a/src/core_atmosphere/stochastic_physics b/src/core_atmosphere/stochastic_physics new file mode 160000 index 0000000000..2f6eace83f --- /dev/null +++ b/src/core_atmosphere/stochastic_physics @@ -0,0 +1 @@ +Subproject commit 2f6eace83f662db2ee16cad8f831c012e7be92f5 diff --git a/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/namelist.atmosphere b/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/namelist.atmosphere new file mode 100644 index 0000000000..23ff5b38d2 --- /dev/null +++ b/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/namelist.atmosphere @@ -0,0 +1,93 @@ +&nhyd_model + config_time_integration_order = 2 + config_dt = 720.0 +! config_dt = 60.0 + config_start_time = '2023-03-10_15:00:00' +! config_run_duration = '0_01:00:00' + config_run_duration = '0_06:00:00' + config_split_dynamics_transport = true + config_number_of_sub_steps = 4 + config_dynamics_split_steps = 3 + config_horiz_mixing = '2d_smagorinsky' + config_visc4_2dsmag = 0.05 + config_scalar_advection = true + config_monotonic = true + config_coef_3rd_order = 0.25 + config_epssm = 0.1 + config_smdiv = 0.1 +/ +&damping + config_mpas_cam_coef = 2.0 + config_rayleigh_damp_u = true + config_zd = 16000.0 + config_xnutr = 0.2 + config_number_cam_damping_levels = 8 +/ +&limited_area + config_apply_lbcs = true +/ +&io + config_pio_num_iotasks = 0 + config_pio_stride = 1 +/ +&decomposition + config_block_decomp_file_prefix = 'graph.info.part.' +/ +&restart + config_do_restart = false +/ +&printout + config_print_global_minmax_vel = true + config_print_detailed_minmax_vel = false +/ +&IAU + config_IAU_option = 'off' + config_IAU_window_length_s = 21600.0 +/ +&physics + config_sst_update = false + config_sstdiurn_update = false + config_frac_seaice = true + config_deepsoiltemp_update = false + config_radtlw_interval = '00:15:00' + config_radtsw_interval = '00:15:00' + config_bucket_update = 'none' + config_physics_suite = 'hrrrv5' + num_soil_layers = 9 + config_mynn_mixnumcon = 0 +/ +&physics_mp_tempo + config_tempo_aerosolaware = true + config_tempo_hailaware = true +/ +&soundings + config_sounding_interval = 'none' +/ +&nam_stochy + do_sppt = true + do_skeb = false +! config_spptint = 60 + config_spptint = 0 +! config_spptint = 720 + config_sppt_1 = 0.8 + config_sppt_2 = 0.0 + config_sppt_3 = 0.0 +! config_sppt_tau_1 = 21600 +! config_sppt_tau_1 = 600 +! config_sppt_tau_1 = 3600 + config_sppt_tau_1 = 10800 + config_sppt_tau_2 = 86400 + config_sppt_tau_3 = 21600 + config_sppt_lscale_1 = 1500000 +! config_sppt_lscale_1 = 150000 + config_sppt_lscale_2 = 1000000 + config_sppt_lscale_3 = 2000000 + config_sppt_logit = true + config_sppt_sfclimit = true + config_iseed_sppt1 = 3 + config_iseed_sppt2 = 0 + config_iseed_sppt3 = 0 + config_sppt_hgt_top1 = 15000 + config_sppt_hgt_top2 = 27000 + config_stochini = false +/ diff --git a/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/stream_list.atmosphere.diagnostics b/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/stream_list.atmosphere.diagnostics new file mode 100644 index 0000000000..048ec8b9d6 --- /dev/null +++ b/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/stream_list.atmosphere.diagnostics @@ -0,0 +1,141 @@ +initial_time +xtime +Time +snowncv +rainncv +graupelncv +prec_acc_c +prec_acc_nc +snow_acc_nc +mslp +relhum_50hPa +relhum_100hPa +relhum_200hPa +relhum_250hPa +relhum_500hPa +relhum_700hPa +relhum_850hPa +relhum_925hPa +dewpoint_50hPa +dewpoint_100hPa +dewpoint_200hPa +dewpoint_250hPa +dewpoint_500hPa +dewpoint_700hPa +dewpoint_850hPa +dewpoint_925hPa +temperature_50hPa +temperature_100hPa +temperature_200hPa +temperature_250hPa +temperature_500hPa +temperature_700hPa +temperature_850hPa +temperature_925hPa +height_50hPa +height_100hPa +height_200hPa +height_250hPa +height_500hPa +height_700hPa +height_850hPa +height_925hPa +uzonal_50hPa +uzonal_100hPa +uzonal_200hPa +uzonal_250hPa +uzonal_500hPa +uzonal_700hPa +uzonal_850hPa +uzonal_925hPa +umeridional_50hPa +umeridional_100hPa +umeridional_200hPa +umeridional_250hPa +umeridional_500hPa +umeridional_700hPa +umeridional_850hPa +umeridional_925hPa +w_50hPa +w_100hPa +w_200hPa +w_250hPa +w_500hPa +w_700hPa +w_850hPa +w_925hPa +vorticity_50hPa +vorticity_100hPa +vorticity_200hPa +vorticity_250hPa +vorticity_500hPa +vorticity_700hPa +vorticity_850hPa +vorticity_925hPa +t_isobaric +t_iso_levels +z_isobaric +z_iso_levels +meanT_500_300 +olrtoa +rainc +rainnc +refl10cm +refl10cm_max +refl10cm_1km +refl10cm_1km_max +precipw +u10 +v10 +q2 +t2m +th2m +cape +cin +lcl +lfc +srh_0_1km +srh_0_3km +uzonal_surface +uzonal_1km +uzonal_6km +umeridional_surface +umeridional_1km +umeridional_6km +temperature_surface +dewpoint_surface +updraft_helicity_max +w_velocity_max +w_velocity_min +w_velocity_mean +wind_speed_level1_max +t_oml +t_oml_initial +t_oml_200m_initial +h_oml +h_oml_initial +hu_oml +hv_oml +updraft_helicity_max_01 +updraft_helicity_max_16 +updraft_helicity_max0_15 +updraft_helicity_max05_15 +updraft_helicity_max_02 +updraft_helicity_max_03 +updraft_helicity_min_01 +updraft_helicity_min_16 +updraft_helicity_min0_15 +updraft_helicity_min05_15 +updraft_helicity_min_02 +updraft_helicity_min_03 +wind02_max +wind02_integral_max +wind_speed_10m_max +wind_speed_10m_mean +u10m_mean +v10m_mean +swdnb +swupt +swddir +swddni +swddif diff --git a/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/stream_list.atmosphere.output b/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/stream_list.atmosphere.output new file mode 100644 index 0000000000..9c151a488d --- /dev/null +++ b/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/stream_list.atmosphere.output @@ -0,0 +1,127 @@ +scalars +latCell +lonCell +xCell +yCell +zCell +indexToCellID +latEdge +lonEdge +xEdge +yEdge +zEdge +indexToEdgeID +latVertex +lonVertex +xVertex +yVertex +zVertex +indexToVertexID +cellsOnEdge +nEdgesOnCell +nEdgesOnEdge +edgesOnCell +edgesOnEdge +weightsOnEdge +dvEdge +dcEdge +angleEdge +areaCell +areaTriangle +cellsOnCell +verticesOnCell +verticesOnEdge +edgesOnVertex +cellsOnVertex +kiteAreasOnVertex +meshDensity +zgrid +fzm +fzp +zz +initial_time +xtime +Time +u +w +pressure +surface_pressure +rho +theta +relhum +divergence +vorticity +ke +uReconstructZonal +uReconstructMeridional +ertel_pv +u_pv +v_pv +theta_pv +vort_pv +iLev_DT +depv_dt_lw +depv_dt_sw +depv_dt_bl +depv_dt_cu +depv_dt_mix +dtheta_dt_mp +depv_dt_mp +depv_dt_diab +depv_dt_fric +depv_dt_diab_pv +depv_dt_fric_pv +i_rainnc +rainnc +precipw +cuprec +i_rainc +rainc +kpbl +hpbl +hfx +qfx +cd +cda +ck +cka +lh +u10 +v10 +q2 +t2m +th2m +gsw +glw +acsnow +pin +ozmixm +mminlu +isice_lu +iswater_lu +xland +skintemp +snow +snowh +sst +vegfra +shdmin +shdmax +canwat +xice +sh2o +smois +tslb +soilt1 +rhosnf +snowfallac +acrunoff +grdflx +ivgtyp +isltyp +lai +sfc_albedo +cldfrac_bl +snowc +snownc +graupelnc diff --git a/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/stream_list.atmosphere.surface b/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/stream_list.atmosphere.surface new file mode 100644 index 0000000000..db54cab459 --- /dev/null +++ b/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/stream_list.atmosphere.surface @@ -0,0 +1,2 @@ +sst +xice diff --git a/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/streams.atmosphere b/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/streams.atmosphere new file mode 100644 index 0000000000..4f19e3696b --- /dev/null +++ b/testing_and_setup/ufs-community/cases/ufscommunity.hrrrv5.gfs.winter.stoch/streams.atmosphere @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + +