From 6647576b05924a594d807f3a122b0771b216b41f Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Fri, 24 Apr 2026 10:14:34 -0400 Subject: [PATCH 1/9] adds polaris cron scripts --- cron/polaris_aurora_cron.sh | 11 ++++++++++ cron/polaris_cron.sh | 40 +++++++++++++++++++++++++++++++++++ cron/polaris_frontier_cron.sh | 11 ++++++++++ cron/polaris_pm-cpu_cron.sh | 11 ++++++++++ cron/polaris_pm-gpu_cron.sh | 11 ++++++++++ 5 files changed, 84 insertions(+) create mode 100755 cron/polaris_aurora_cron.sh create mode 100755 cron/polaris_cron.sh create mode 100755 cron/polaris_frontier_cron.sh create mode 100755 cron/polaris_pm-cpu_cron.sh create mode 100755 cron/polaris_pm-gpu_cron.sh diff --git a/cron/polaris_aurora_cron.sh b/cron/polaris_aurora_cron.sh new file mode 100755 index 0000000..ff9efbf --- /dev/null +++ b/cron/polaris_aurora_cron.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +HERE=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +# top directory of all Polaris cronjob work +export POLARIS_CRON_ROOT="/lus/flare/projects/$PROJECT/$USER/polaris_scratch/cron" + +mkdir -p $POLARIS_CRON_ROOT + +# launch polaris cronjob +exec bash $HERE/polaris_cron.sh diff --git a/cron/polaris_cron.sh b/cron/polaris_cron.sh new file mode 100755 index 0000000..c4fa7cf --- /dev/null +++ b/cron/polaris_cron.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Configuration +export POLARIS_ROOT=$POLARIS_CRON_ROOT/polaris +REMOTE_URL="https://github.com/E3SM-Project/polaris.git" +BRANCH="main" + +# Check existence and handle repository state +if [ ! -d "$POLARIS_ROOT/.git" ]; then + echo "Repository not found. Cloning..." + git clone -b "$BRANCH" "$REMOTE_URL" "$POLARIS_ROOT" + cd "$POLARIS_ROOT" || exit +else + echo "Repository exists. Updating to latest remote state..." + cd "$POLARIS_ROOT" || exit + + # Ensure we are on the correct branch and sync with origin + git fetch origin + git checkout "$BRANCH" + git reset --hard "origin/$BRANCH" +fi + +# Update specific submodules recursively +echo "Updating submodules..." +git submodule update --init --recursive jigsaw-python +git submodule update --init e3sm_submodules/Omega +pushd e3sm_submodules/Omega +git submodule update --init --recursive externals/ekat externals/scorpio cime components/omega/external +popd + +# Launch the final script with all passed arguments +LAUNCH_SCRIPT="$POLARIS_ROOT/cron-scripts/launch_all.sh" + +if [ -f "$LAUNCH_SCRIPT" ]; then + echo "Launching: $LAUNCH_SCRIPT $*" + exec bash "$LAUNCH_SCRIPT" "$@" +else + echo "Error: Launch script not found at $LAUNCH_SCRIPT" + exit 1 +fi diff --git a/cron/polaris_frontier_cron.sh b/cron/polaris_frontier_cron.sh new file mode 100755 index 0000000..0eb3e5e --- /dev/null +++ b/cron/polaris_frontier_cron.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +HERE=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +# top directory of all Polaris cronjob work +export POLARIS_CRON_ROOT="/lustre/orion/cli115/proj-shared/$ENV{USER}/polaris_scratch/cron" + +mkdir -p $POLARIS_CRON_ROOT + +# launch polaris cronjob +exec bash $HERE/polaris_cron.sh diff --git a/cron/polaris_pm-cpu_cron.sh b/cron/polaris_pm-cpu_cron.sh new file mode 100755 index 0000000..fe2b205 --- /dev/null +++ b/cron/polaris_pm-cpu_cron.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +HERE=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +# top directory of all Polaris cronjob work +export POLARIS_CRON_ROOT="$PSCRATCH/polaris_scratch/cron/pm-cpu" + +mkdir -p $POLARIS_CRON_ROOT + +# launch polaris cronjob +exec bash $HERE/polaris_cron.sh diff --git a/cron/polaris_pm-gpu_cron.sh b/cron/polaris_pm-gpu_cron.sh new file mode 100755 index 0000000..8cb8ab2 --- /dev/null +++ b/cron/polaris_pm-gpu_cron.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +HERE=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +# top directory of all Polaris cronjob work +export POLARIS_CRON_ROOT="$PSCRATCH/polaris_scratch/cron/pm-gpu" + +mkdir -p $POLARIS_CRON_ROOT + +# launch polaris cronjob +exec bash $HERE/polaris_cron.sh From da5c8432a6ca471fcf890489bdc099a0ac4bb7da Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Fri, 24 Apr 2026 11:24:27 -0400 Subject: [PATCH 2/9] add arguments for pm-cpu and pm-gpu --- cron/polaris_pm-cpu_cron.sh | 2 +- cron/polaris_pm-gpu_cron.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cron/polaris_pm-cpu_cron.sh b/cron/polaris_pm-cpu_cron.sh index fe2b205..c1d8105 100755 --- a/cron/polaris_pm-cpu_cron.sh +++ b/cron/polaris_pm-cpu_cron.sh @@ -8,4 +8,4 @@ export POLARIS_CRON_ROOT="$PSCRATCH/polaris_scratch/cron/pm-cpu" mkdir -p $POLARIS_CRON_ROOT # launch polaris cronjob -exec bash $HERE/polaris_cron.sh +exec bash $HERE/polaris_cron.sh -m pm-cpu diff --git a/cron/polaris_pm-gpu_cron.sh b/cron/polaris_pm-gpu_cron.sh index 8cb8ab2..ec449a2 100755 --- a/cron/polaris_pm-gpu_cron.sh +++ b/cron/polaris_pm-gpu_cron.sh @@ -8,4 +8,4 @@ export POLARIS_CRON_ROOT="$PSCRATCH/polaris_scratch/cron/pm-gpu" mkdir -p $POLARIS_CRON_ROOT # launch polaris cronjob -exec bash $HERE/polaris_cron.sh +exec bash $HERE/polaris_cron.sh -m pm-gpu From e09494a28d88bdbbd496f8aad78329412e44190f Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Fri, 24 Apr 2026 15:31:26 +0000 Subject: [PATCH 3/9] adjust aurora path --- cron/polaris_aurora_cron.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/polaris_aurora_cron.sh b/cron/polaris_aurora_cron.sh index ff9efbf..fc2f701 100755 --- a/cron/polaris_aurora_cron.sh +++ b/cron/polaris_aurora_cron.sh @@ -3,7 +3,7 @@ HERE=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) # top directory of all Polaris cronjob work -export POLARIS_CRON_ROOT="/lus/flare/projects/$PROJECT/$USER/polaris_scratch/cron" +export POLARIS_CRON_ROOT="/lus/flare/projects/E3SM_Dec/$USER/polaris_scratch/cron" mkdir -p $POLARIS_CRON_ROOT From 15a455bf4817855c13eb65f1c17cae61e3f8adad Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Mon, 27 Apr 2026 09:33:33 -0400 Subject: [PATCH 4/9] fixed a bug in polaris_frontier_cron.sh on using USER env var --- cron/polaris_frontier_cron.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/polaris_frontier_cron.sh b/cron/polaris_frontier_cron.sh index 0eb3e5e..376addc 100755 --- a/cron/polaris_frontier_cron.sh +++ b/cron/polaris_frontier_cron.sh @@ -3,7 +3,7 @@ HERE=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) # top directory of all Polaris cronjob work -export POLARIS_CRON_ROOT="/lustre/orion/cli115/proj-shared/$ENV{USER}/polaris_scratch/cron" +export POLARIS_CRON_ROOT="/lustre/orion/cli115/proj-shared/${USER}/polaris_scratch/cron" mkdir -p $POLARIS_CRON_ROOT From a94808df14dc1c5e8749e041df9d5d532719780a Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Wed, 29 Apr 2026 15:56:02 -0400 Subject: [PATCH 5/9] download polaris repo at every test --- cron/polaris_cron.sh | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/cron/polaris_cron.sh b/cron/polaris_cron.sh index c4fa7cf..785e688 100755 --- a/cron/polaris_cron.sh +++ b/cron/polaris_cron.sh @@ -2,23 +2,15 @@ # Configuration export POLARIS_ROOT=$POLARIS_CRON_ROOT/polaris -REMOTE_URL="https://github.com/E3SM-Project/polaris.git" -BRANCH="main" +#REMOTE_URL="https://github.com/E3SM-Project/polaris.git" +#BRANCH="main" +REMOTE_URL="https://github.com/grnydawn/polaris.git" +BRANCH="ykim/copilot/cron-scripts" -# Check existence and handle repository state -if [ ! -d "$POLARIS_ROOT/.git" ]; then - echo "Repository not found. Cloning..." - git clone -b "$BRANCH" "$REMOTE_URL" "$POLARIS_ROOT" - cd "$POLARIS_ROOT" || exit -else - echo "Repository exists. Updating to latest remote state..." - cd "$POLARIS_ROOT" || exit - - # Ensure we are on the correct branch and sync with origin - git fetch origin - git checkout "$BRANCH" - git reset --hard "origin/$BRANCH" -fi +rm -rf ${POLARIS_ROOT} + +git clone -b "$BRANCH" "$REMOTE_URL" "$POLARIS_ROOT" +cd "$POLARIS_ROOT" || exit # Update specific submodules recursively echo "Updating submodules..." From 1b44664f5ff2cb772d8e1d6110bb858349a09e88 Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Mon, 4 May 2026 13:36:07 -0400 Subject: [PATCH 6/9] remove slurm memory restriction on pm-gpu --- cron/polaris_pm-gpu_cron.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cron/polaris_pm-gpu_cron.sh b/cron/polaris_pm-gpu_cron.sh index ec449a2..6697a36 100755 --- a/cron/polaris_pm-gpu_cron.sh +++ b/cron/polaris_pm-gpu_cron.sh @@ -7,5 +7,10 @@ export POLARIS_CRON_ROOT="$PSCRATCH/polaris_scratch/cron/pm-gpu" mkdir -p $POLARIS_CRON_ROOT +if [[ ! -z "${SLURM_MEM_PER_CPU}" ]]; then +unset SLURM_MEM_PER_CPU +unset SLURM_OPEN_MODE +fi + # launch polaris cronjob exec bash $HERE/polaris_cron.sh -m pm-gpu From c02e834f98b8e28e167dcf6ed0888f0bc8e07d8a Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Tue, 5 May 2026 11:46:43 -0700 Subject: [PATCH 7/9] add polaris cron script for Chrysalis --- cron/polaris_chrysalis_cron.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 cron/polaris_chrysalis_cron.sh diff --git a/cron/polaris_chrysalis_cron.sh b/cron/polaris_chrysalis_cron.sh new file mode 100755 index 0000000..ab30ee7 --- /dev/null +++ b/cron/polaris_chrysalis_cron.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +HERE=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) + +# top directory of all Polaris cronjob work +export POLARIS_CRON_ROOT="/lcrc/group/e3sm/$USER/scratch/cron" + +mkdir -p $POLARIS_CRON_ROOT + +# launch polaris cronjob +exec bash $HERE/polaris_cron.sh From c23d859042310db7570fd54dcbfa354e387776b0 Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Wed, 6 May 2026 10:21:08 -0700 Subject: [PATCH 8/9] update polaris branch name for cron job --- cron/polaris_cron.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron/polaris_cron.sh b/cron/polaris_cron.sh index 785e688..b4e47f4 100755 --- a/cron/polaris_cron.sh +++ b/cron/polaris_cron.sh @@ -5,7 +5,7 @@ export POLARIS_ROOT=$POLARIS_CRON_ROOT/polaris #REMOTE_URL="https://github.com/E3SM-Project/polaris.git" #BRANCH="main" REMOTE_URL="https://github.com/grnydawn/polaris.git" -BRANCH="ykim/copilot/cron-scripts" +BRANCH="ykim/cron-scripts" rm -rf ${POLARIS_ROOT} From fdc34597686ead089cc9d8437f5bf93ae7b28aa7 Mon Sep 17 00:00:00 2001 From: Youngsung Kim Date: Wed, 6 May 2026 12:15:08 -0700 Subject: [PATCH 9/9] use polaris main branch for nightly testing --- cron/polaris_cron.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cron/polaris_cron.sh b/cron/polaris_cron.sh index b4e47f4..7b5bc67 100755 --- a/cron/polaris_cron.sh +++ b/cron/polaris_cron.sh @@ -2,10 +2,8 @@ # Configuration export POLARIS_ROOT=$POLARIS_CRON_ROOT/polaris -#REMOTE_URL="https://github.com/E3SM-Project/polaris.git" -#BRANCH="main" -REMOTE_URL="https://github.com/grnydawn/polaris.git" -BRANCH="ykim/cron-scripts" +REMOTE_URL="https://github.com/E3SM-Project/polaris.git" +BRANCH="main" rm -rf ${POLARIS_ROOT}