From 533173632daf0a9d54d3eeace1edcdae89816b78 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Thu, 21 Dec 2023 16:53:17 -0500 Subject: [PATCH 01/27] trigger circleci workflow on pr --- .circleci/config.yml | 47 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 656b8c7..03b4d50 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: parameters: ssh: type: string - default: "ssh -o StrictHostKeyChecking=no $SCOPULI_RUNNER_SSH_USER@$SCOPULI_RUNNER_SSH_HOST SCOPULI_RUNNER_DIR=$SCOPULI_RUNNER_DIR" + default: "ssh -o StrictHostKeyChecking=no $JOBRUNNER_RUNNER_SSH_USER@$JOBRUNNER_RUNNER_SSH_HOST JOBRUNNER_RUNNER_DIR=$JOBRUNNER_RUNNER_DIR" scp: type: string default: "scp -o StrictHostKeyChecking=no" @@ -30,11 +30,15 @@ jobs: - run: name: Pull Latest Head of Main command: | - << parameters.ssh >> "cd $SCOPULI_RUNNER_DIR/scopuli && git checkout $SCOPULI_ENV && git pull" + << parameters.ssh >> "cd $JOBRUNNER_RUNNER_DIR/jobrunner && git checkout main && git pull" + - run: + name: Pull Latest Head of Aethervest Main + command: | + << parameters.ssh >> "cd $AETHERVEST_DIR && git clone $AETHERVEST_REPO_SSH && cd $aethervest/ && git checkout main && git pull" - run: name: Set up server's environment variables description: | - Sets up the server's environmnet variables. Avoid using ssh to source the variables in this step. + Sets up the server's environment variables. Avoid using ssh to source the variables in this step. It can cause an exit 255 since the scp command precedes it. command: | echo "export SCOPULI_PROD_PG_HOST=${SCOPULI_PROD_PG_HOST}" >> .env @@ -47,7 +51,30 @@ jobs: - run: name: Update the cron schedule command: | - << parameters.ssh >> "bash $SCOPULI_RUNNER_DIR/scopuli/scheduler/schedule.sh" + << parameters.ssh >> "bash $JOBRUNNER_RUNNER_DIR/jobrunner/schedule.sh" + - run: + name: Check crontab + commands: | + - << parameters.ssh >> + - > + if grep -q "0 0 1 */1 * python aethervest/statcan/cpi_monthly_1810000401/loader.py" <<< "crontab -l"; then + echo "Monthly CPI loader job found in crontab." + else + echo "Monthly CPI loader job not found in crontab." + fi + - > + if grep -q "0 0 1 */1 * python aethervest/statcan/food_prices_1810024501/loader.py" <<< "crontab -l"; then + echo "Food price loader job found in crontab." + else + echo "Food price loader job not found in crontab" + fi + test: + steps: + - run: + name: Test pull request trigger + commands: + echo "PR opened" + workflows: deploy-to-prod: jobs: @@ -57,3 +84,15 @@ workflows: filters: branches: only: main + pull-request: + jobs: + - test: + context: + - aethersphere-prod + + +# environment variables: +# - JOBRUNNER_RUNNER_SSH_USER@$JOBRUNNER_RUNNER_SSH_HOST +# - JOBRUNNER_ENV = main, AETHERVEST_ENV = main +# - AETHERVEST_DIR = /home/circleci/jobrunner/ +# - AETHERVEST_REPO_SSH = git@github.com:tm41m/aethervest.git \ No newline at end of file From 6a0a241539c3182104b23c022406f0baa0aa6239 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Thu, 11 Jan 2024 15:53:12 -0500 Subject: [PATCH 02/27] initial commit --- .circleci/config.yml | 52 +++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 03b4d50..9671035 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,26 +54,26 @@ jobs: << parameters.ssh >> "bash $JOBRUNNER_RUNNER_DIR/jobrunner/schedule.sh" - run: name: Check crontab - commands: | - - << parameters.ssh >> - - > - if grep -q "0 0 1 */1 * python aethervest/statcan/cpi_monthly_1810000401/loader.py" <<< "crontab -l"; then - echo "Monthly CPI loader job found in crontab." - else - echo "Monthly CPI loader job not found in crontab." - fi - - > - if grep -q "0 0 1 */1 * python aethervest/statcan/food_prices_1810024501/loader.py" <<< "crontab -l"; then - echo "Food price loader job found in crontab." - else - echo "Food price loader job not found in crontab" - fi + command: | + << parameters.ssh >> + if grep -q "0 0 1 */1 * python aethervest/statcan/cpi_monthly_1810000401/loader.py" <<< "crontab -l"; then + echo "Monthly CPI loader job found in crontab." + else + echo "Monthly CPI loader job not found in crontab." + fi + if grep -q "0 0 1 */1 * python aethervest/statcan/food_prices_1810024501/loader.py" <<< "crontab -l"; then + echo "Food price loader job found in crontab." + else + echo "Food price loader job not found in crontab" + fi test: + docker: + - image: cimg/base:2023.06 steps: - run: - name: Test pull request trigger - commands: - echo "PR opened" + name: Test push to remote main trigger + command: | + echo "new commit to main" workflows: deploy-to-prod: @@ -84,15 +84,13 @@ workflows: filters: branches: only: main - pull-request: + push-to-remote-main: jobs: - test: - context: - - aethersphere-prod - - -# environment variables: -# - JOBRUNNER_RUNNER_SSH_USER@$JOBRUNNER_RUNNER_SSH_HOST -# - JOBRUNNER_ENV = main, AETHERVEST_ENV = main -# - AETHERVEST_DIR = /home/circleci/jobrunner/ -# - AETHERVEST_REPO_SSH = git@github.com:tm41m/aethervest.git \ No newline at end of file + filters: + branches: + only: ashannon/jobrunner/dispatch_jobs + - deploy: + filters: + branches: + only: main \ No newline at end of file From 3bae30fd56d2283af1183a354a99826fb47a3de2 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Thu, 11 Jan 2024 15:57:29 -0500 Subject: [PATCH 03/27] job to list crontab --- .circleci/config.yml | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9671035..23c75e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,17 +55,17 @@ jobs: - run: name: Check crontab command: | - << parameters.ssh >> - if grep -q "0 0 1 */1 * python aethervest/statcan/cpi_monthly_1810000401/loader.py" <<< "crontab -l"; then - echo "Monthly CPI loader job found in crontab." - else - echo "Monthly CPI loader job not found in crontab." - fi - if grep -q "0 0 1 */1 * python aethervest/statcan/food_prices_1810024501/loader.py" <<< "crontab -l"; then - echo "Food price loader job found in crontab." - else - echo "Food price loader job not found in crontab" - fi + << parameters.ssh >> ""bash crontab -l" +# if grep -q "0 0 1 */1 * python aethervest/statcan/cpi_monthly_1810000401/loader.py" <<< "crontab -l"; then +# echo "Monthly CPI loader job found in crontab." +# else +# echo "Monthly CPI loader job not found in crontab." +# fi +# if grep -q "0 0 1 */1 * python aethervest/statcan/food_prices_1810024501/loader.py" <<< "crontab -l"; then +# echo "Food price loader job found in crontab." +# else +# echo "Food price loader job not found in crontab" +# fi test: docker: - image: cimg/base:2023.06 @@ -93,4 +93,7 @@ workflows: - deploy: filters: branches: - only: main \ No newline at end of file + only: main + + +# asdasd \ No newline at end of file From 0607b7fbe132475639104894f58ea46a0a9fe544 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Thu, 11 Jan 2024 15:59:02 -0500 Subject: [PATCH 04/27] change workflow branch --- .circleci/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 23c75e0..fdab9ab 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,7 +93,5 @@ workflows: - deploy: filters: branches: - only: main - + only: ashannon/jobrunner/dispatch_jobs -# asdasd \ No newline at end of file From 367fa0e81fdefd4108b4ebb9a26c7efee75df164 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Mon, 15 Jan 2024 16:37:00 -0500 Subject: [PATCH 05/27] change context --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fdab9ab..ed0257f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,7 +80,7 @@ workflows: jobs: - deploy: context: - - aethersphere-prod + - jobrunner-prod filters: branches: only: main @@ -91,6 +91,8 @@ workflows: branches: only: ashannon/jobrunner/dispatch_jobs - deploy: + context: + - jobrunner-prod filters: branches: only: ashannon/jobrunner/dispatch_jobs From 0a2b513518b6d2d8098eae5f52368ba791c5730e Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Mon, 15 Jan 2024 16:59:27 -0500 Subject: [PATCH 06/27] test context --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ed0257f..1d31ac1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,7 +30,8 @@ jobs: - run: name: Pull Latest Head of Main command: | - << parameters.ssh >> "cd $JOBRUNNER_RUNNER_DIR/jobrunner && git checkout main && git pull" + echo "$JOBRUNNER_RUNNER_DIR" +# < parameters.ssh >> "cd $JOBRUNNER_RUNNER_DIR/jobrunner && git checkout main && git pull" - run: name: Pull Latest Head of Aethervest Main command: | From 2ca7cca7931c3f4fa10f91a9b37c6c7ba6048e7e Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Mon, 15 Jan 2024 17:46:43 -0500 Subject: [PATCH 07/27] display env --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d31ac1..941efb3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,6 +27,9 @@ jobs: name: Add SSH forwarding command: | eval `ssh-agent -s` && ssh-add -k + - run: + name: Print environment vars + command: printenv - run: name: Pull Latest Head of Main command: | From 0fbde7853cea23b0db2ecc771b962d9c75b7dcb4 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Tue, 16 Jan 2024 13:08:45 -0500 Subject: [PATCH 08/27] rename var --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 941efb3..9dac348 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: parameters: ssh: type: string - default: "ssh -o StrictHostKeyChecking=no $JOBRUNNER_RUNNER_SSH_USER@$JOBRUNNER_RUNNER_SSH_HOST JOBRUNNER_RUNNER_DIR=$JOBRUNNER_RUNNER_DIR" + default: "ssh -o StrictHostKeyChecking=no $JOBRUNNER_SSH_USER@$JOBRUNNER_SSH_HOST JOBRUNNER_DIR=$JOBRUNNER_DIR" scp: type: string default: "scp -o StrictHostKeyChecking=no" @@ -33,8 +33,8 @@ jobs: - run: name: Pull Latest Head of Main command: | - echo "$JOBRUNNER_RUNNER_DIR" -# < parameters.ssh >> "cd $JOBRUNNER_RUNNER_DIR/jobrunner && git checkout main && git pull" + echo "$JOBRUNNER_DIR" +# < parameters.ssh >> "cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" - run: name: Pull Latest Head of Aethervest Main command: | @@ -55,7 +55,7 @@ jobs: - run: name: Update the cron schedule command: | - << parameters.ssh >> "bash $JOBRUNNER_RUNNER_DIR/jobrunner/schedule.sh" + << parameters.ssh >> "bash $JOBRUNNER_DIR/jobrunner/schedule.sh" - run: name: Check crontab command: | From 086c46f27d2768faabc9e30db0f1ae8aa76adce3 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Tue, 16 Jan 2024 13:10:50 -0500 Subject: [PATCH 09/27] change context to aethersphere --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9dac348..2c00312 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,7 +84,7 @@ workflows: jobs: - deploy: context: - - jobrunner-prod + - aethersphere-prod filters: branches: only: main @@ -96,7 +96,7 @@ workflows: only: ashannon/jobrunner/dispatch_jobs - deploy: context: - - jobrunner-prod + - aethersphere-prod filters: branches: only: ashannon/jobrunner/dispatch_jobs From 045ea9620e1390798cc20a1eb77279c0eeef713c Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Tue, 16 Jan 2024 13:17:43 -0500 Subject: [PATCH 10/27] comment aethervest pull --- .circleci/config.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2c00312..557ad1e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,12 +33,11 @@ jobs: - run: name: Pull Latest Head of Main command: | - echo "$JOBRUNNER_DIR" -# < parameters.ssh >> "cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" + << parameters.ssh >> "cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" - run: name: Pull Latest Head of Aethervest Main command: | - << parameters.ssh >> "cd $AETHERVEST_DIR && git clone $AETHERVEST_REPO_SSH && cd $aethervest/ && git checkout main && git pull" +# << parameters.ssh >> "cd $AETHERVEST_DIR && git clone $AETHERVEST_REPO_SSH && cd $aethervest/ && git checkout main && git pull" - run: name: Set up server's environment variables description: | From dcd98a48e25f6c4d9c987952f57597372f1d5fc3 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Tue, 16 Jan 2024 13:18:52 -0500 Subject: [PATCH 11/27] comment aethervest pull --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 557ad1e..ba58879 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,9 +34,9 @@ jobs: name: Pull Latest Head of Main command: | << parameters.ssh >> "cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" - - run: - name: Pull Latest Head of Aethervest Main - command: | +# - run: +# name: Pull Latest Head of Aethervest Main +# command: | # << parameters.ssh >> "cd $AETHERVEST_DIR && git clone $AETHERVEST_REPO_SSH && cd $aethervest/ && git checkout main && git pull" - run: name: Set up server's environment variables From 1602cc1e59114b6e1185cac8a4cc27f6e9693469 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Tue, 16 Jan 2024 13:21:11 -0500 Subject: [PATCH 12/27] mkdir jobrunner/ --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba58879..5c6a2f6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,7 @@ jobs: - run: name: Pull Latest Head of Main command: | - << parameters.ssh >> "cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" + << parameters.ssh >> "mkdir /jobrunner/ cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" # - run: # name: Pull Latest Head of Aethervest Main # command: | From cfed8bf94304630140845d5baeea79468dffdf96 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Tue, 16 Jan 2024 13:23:34 -0500 Subject: [PATCH 13/27] mkdir jobrunner/ --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5c6a2f6..fcdda3d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,7 @@ jobs: - run: name: Pull Latest Head of Main command: | - << parameters.ssh >> "mkdir /jobrunner/ cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" + << parameters.ssh >> "mkdir $JOBRUNNER_DIR/jobrunner cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" # - run: # name: Pull Latest Head of Aethervest Main # command: | From 8cfd3f2f0abd116ff0f7e360bc819fc595e3c35a Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Tue, 16 Jan 2024 13:25:42 -0500 Subject: [PATCH 14/27] ampersands after mkdir --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fcdda3d..29a96a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,7 @@ jobs: - run: name: Pull Latest Head of Main command: | - << parameters.ssh >> "mkdir $JOBRUNNER_DIR/jobrunner cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" + << parameters.ssh >> "mkdir $JOBRUNNER_DIR/jobrunner && cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" # - run: # name: Pull Latest Head of Aethervest Main # command: | From a8207f209eaad0ee00b79274ef10fa7494981cdb Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Tue, 16 Jan 2024 13:36:26 -0500 Subject: [PATCH 15/27] remove mkdir --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 29a96a7..ba58879 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,7 @@ jobs: - run: name: Pull Latest Head of Main command: | - << parameters.ssh >> "mkdir $JOBRUNNER_DIR/jobrunner && cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" + << parameters.ssh >> "cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" # - run: # name: Pull Latest Head of Aethervest Main # command: | From 3b8a25a9664b2f1a8b964e86616391f9ccdcced2 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Tue, 16 Jan 2024 13:51:38 -0500 Subject: [PATCH 16/27] fix crontab list --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ba58879..f5ec1b4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,7 +58,7 @@ jobs: - run: name: Check crontab command: | - << parameters.ssh >> ""bash crontab -l" + << parameters.ssh >> "bash crontab -l" # if grep -q "0 0 1 */1 * python aethervest/statcan/cpi_monthly_1810000401/loader.py" <<< "crontab -l"; then # echo "Monthly CPI loader job found in crontab." # else From 279d081c2d6f6aaafa6e570f266b790fd3238120 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Tue, 16 Jan 2024 13:53:03 -0500 Subject: [PATCH 17/27] crontab list --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f5ec1b4..3eebd05 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,7 +58,7 @@ jobs: - run: name: Check crontab command: | - << parameters.ssh >> "bash crontab -l" + << parameters.ssh >> "crontab -l" # if grep -q "0 0 1 */1 * python aethervest/statcan/cpi_monthly_1810000401/loader.py" <<< "crontab -l"; then # echo "Monthly CPI loader job found in crontab." # else From cb55bc119ead3eb4d1fbb9f9a7650cddc32411f4 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Tue, 16 Jan 2024 15:10:19 -0500 Subject: [PATCH 18/27] scopuli submodule --- .gitmodules | 3 +++ scopuli | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 scopuli diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..107d2ac --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "scopuli"] + path = scopuli + url = git@github.com:tm41m/scopuli.git diff --git a/scopuli b/scopuli new file mode 160000 index 0000000..901a2ab --- /dev/null +++ b/scopuli @@ -0,0 +1 @@ +Subproject commit 901a2ab608c60e03bc0497f2e3e6eb61e5b1cddb From d1996bdfd5b85cefb75d06b14729a3202f6883d8 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Wed, 17 Jan 2024 13:10:33 -0500 Subject: [PATCH 19/27] ls step --- .circleci/config.yml | 8 ++++---- schedule.sh | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3eebd05..02633ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,10 +34,6 @@ jobs: name: Pull Latest Head of Main command: | << parameters.ssh >> "cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" -# - run: -# name: Pull Latest Head of Aethervest Main -# command: | -# << parameters.ssh >> "cd $AETHERVEST_DIR && git clone $AETHERVEST_REPO_SSH && cd $aethervest/ && git checkout main && git pull" - run: name: Set up server's environment variables description: | @@ -51,6 +47,10 @@ jobs: echo "export SCOPULI_ENV=${SCOPULI_ENV}" >> .env echo "export SCOPULI_RUNNER_DIR=${SCOPULI_RUNNER_DIR}" >> .env << parameters.scp >> .env $SCOPULI_RUNNER_SSH_USER@$SCOPULI_RUNNER_SSH_HOST:/home/circleci/.env + - run: + name: Display file structure + command: | + "ls" - run: name: Update the cron schedule command: | diff --git a/schedule.sh b/schedule.sh index 2fc9f3c..93e56fd 100644 --- a/schedule.sh +++ b/schedule.sh @@ -4,8 +4,7 @@ crontab -r # dbt run -dbt_run="0 0 */2 * * docker run --env-file /home/circleci/.env --network=host --mount type=bind,source=/home/circleci/scopuli/ganymede/,target=/usr/app --mount type=bind,source=/home/circleci/scopuli/,target=/root/.dbt/ ghcr.io/dbt-labs/dbt-postgres:1.5.2 run --target prod" -cmd_1="0 0 */2 * * pwd" +dbt_run="0 0 */2 * * docker run --env-file /home/circleci/.env --network=host --mount type=bind,source=/home/circleci/jobrunner/scopuli/ganymede/,target=/usr/app --mount type=bind,source=/home/circleci/jobrunner/scopuli/,target=/root/.dbt/ ghcr.io/dbt-labs/dbt-postgres:1.5.2 run --target prod" # Create a temporary file to store the cron job entries temp_file=$(mktemp) From eb8e3533d4e79d0b4502e7a29e5885fef8fab3f2 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Wed, 17 Jan 2024 13:25:13 -0500 Subject: [PATCH 20/27] pwd --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 02633ee..8f4d489 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,7 @@ jobs: - run: name: Display file structure command: | - "ls" + "ls && pwd" - run: name: Update the cron schedule command: | From d735661bc1bc568855fa49ba24624b7a1a6043fb Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Wed, 17 Jan 2024 13:28:02 -0500 Subject: [PATCH 21/27] pwd retry --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8f4d489..ab8446f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,8 @@ jobs: - run: name: Display file structure command: | - "ls && pwd" + ls && + pwd - run: name: Update the cron schedule command: | From 1c981eed0e0fdd308e5f642d552e787901d61ca0 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Wed, 17 Jan 2024 13:33:50 -0500 Subject: [PATCH 22/27] check crontab script --- .circleci/config.yml | 16 +++++----------- check_crontab.sh | 5 +++++ 2 files changed, 10 insertions(+), 11 deletions(-) create mode 100644 check_crontab.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index ab8446f..e4ba362 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,19 +57,13 @@ jobs: command: | << parameters.ssh >> "bash $JOBRUNNER_DIR/jobrunner/schedule.sh" - run: - name: Check crontab + name: List crontab command: | << parameters.ssh >> "crontab -l" -# if grep -q "0 0 1 */1 * python aethervest/statcan/cpi_monthly_1810000401/loader.py" <<< "crontab -l"; then -# echo "Monthly CPI loader job found in crontab." -# else -# echo "Monthly CPI loader job not found in crontab." -# fi -# if grep -q "0 0 1 */1 * python aethervest/statcan/food_prices_1810024501/loader.py" <<< "crontab -l"; then -# echo "Food price loader job found in crontab." -# else -# echo "Food price loader job not found in crontab" -# fi + - run: + name: Check crontab + command: | + << parameters.ssh >> "bash $JOBRUNNER_DIR/jobrunner/check_crontab.sh" test: docker: - image: cimg/base:2023.06 diff --git a/check_crontab.sh b/check_crontab.sh new file mode 100644 index 0000000..e150ec4 --- /dev/null +++ b/check_crontab.sh @@ -0,0 +1,5 @@ +if grep -q "0 0 */2 * * docker run --env-file /home/circleci/.env --network=host --mount type=bind,source=/home/circleci/jobrunner/scopuli/ganymede/,target=/usr/app --mount type=bind,source=/home/circleci/jobrunner/scopuli/,target=/root/.dbt/ ghcr.io/dbt-labs/dbt-postgres:1.5.2 run --target prod" <<< "crontab -l"; then + echo "Correct pathing" +else + echo "Error, incorrect pathing" +fi From 4c65a2e81a6cb48834a5d4f663cc5c4eb9a09e88 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Wed, 17 Jan 2024 13:43:58 -0500 Subject: [PATCH 23/27] change directory --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4ba362..d355c97 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,7 +63,7 @@ jobs: - run: name: Check crontab command: | - << parameters.ssh >> "bash $JOBRUNNER_DIR/jobrunner/check_crontab.sh" + << parameters.ssh >> "bash $JOBRUNNER_DIR/project/check_crontab.sh" test: docker: - image: cimg/base:2023.06 From 39712e3cc2c9316f65e577df0a515970fdc7b206 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Wed, 17 Jan 2024 15:08:57 -0500 Subject: [PATCH 24/27] bash header --- .circleci/config.yml | 2 +- check_crontab.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d355c97..e4ba362 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,7 +63,7 @@ jobs: - run: name: Check crontab command: | - << parameters.ssh >> "bash $JOBRUNNER_DIR/project/check_crontab.sh" + << parameters.ssh >> "bash $JOBRUNNER_DIR/jobrunner/check_crontab.sh" test: docker: - image: cimg/base:2023.06 diff --git a/check_crontab.sh b/check_crontab.sh index e150ec4..7ae1c42 100644 --- a/check_crontab.sh +++ b/check_crontab.sh @@ -1,3 +1,5 @@ +#!/bin/bash + if grep -q "0 0 */2 * * docker run --env-file /home/circleci/.env --network=host --mount type=bind,source=/home/circleci/jobrunner/scopuli/ganymede/,target=/usr/app --mount type=bind,source=/home/circleci/jobrunner/scopuli/,target=/root/.dbt/ ghcr.io/dbt-labs/dbt-postgres:1.5.2 run --target prod" <<< "crontab -l"; then echo "Correct pathing" else From 6c7dd35369e833c22012b8cc95e6872212599cd7 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Wed, 17 Jan 2024 15:24:31 -0500 Subject: [PATCH 25/27] checkout feature branch --- .circleci/config.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4ba362..10e0e0d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,7 @@ jobs: - run: name: Pull Latest Head of Main command: | - << parameters.ssh >> "cd $JOBRUNNER_DIR/jobrunner && git checkout main && git pull" + << parameters.ssh >> "cd $JOBRUNNER_DIR/jobrunner && git checkout ashannon/jobrunner/dispatch_jobs && git pull" - run: name: Set up server's environment variables description: | @@ -47,11 +47,6 @@ jobs: echo "export SCOPULI_ENV=${SCOPULI_ENV}" >> .env echo "export SCOPULI_RUNNER_DIR=${SCOPULI_RUNNER_DIR}" >> .env << parameters.scp >> .env $SCOPULI_RUNNER_SSH_USER@$SCOPULI_RUNNER_SSH_HOST:/home/circleci/.env - - run: - name: Display file structure - command: | - ls && - pwd - run: name: Update the cron schedule command: | From d6c3c1a07d48122bda8dbc323fafd578ae34e060 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Wed, 17 Jan 2024 15:39:31 -0500 Subject: [PATCH 26/27] modified check_crontab --- check_crontab.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/check_crontab.sh b/check_crontab.sh index 7ae1c42..caee241 100644 --- a/check_crontab.sh +++ b/check_crontab.sh @@ -1,6 +1,10 @@ #!/bin/bash -if grep -q "0 0 */2 * * docker run --env-file /home/circleci/.env --network=host --mount type=bind,source=/home/circleci/jobrunner/scopuli/ganymede/,target=/usr/app --mount type=bind,source=/home/circleci/jobrunner/scopuli/,target=/root/.dbt/ ghcr.io/dbt-labs/dbt-postgres:1.5.2 run --target prod" <<< "crontab -l"; then +target_job="0 0 */2 * * docker run --env-file /home/circleci/.env --network=host --mount type=bind,source=/home/circleci/jobrunner/scopuli/ganymede/,target=/usr/app --mount type=bind,source=/home/circleci/jobrunner/scopuli/,target=/root/.dbt/ ghcr.io/dbt-labs/dbt-postgres:1.5.2 run --target prod" + +crontab_content=$(crontab -l) + +if [[ $crontab_content == *"$target_job"* ]]; then echo "Correct pathing" else echo "Error, incorrect pathing" From 5811e2850e457100b4637b5bce3264306d361f55 Mon Sep 17 00:00:00 2001 From: Andrew Shannon Date: Wed, 17 Jan 2024 15:42:58 -0500 Subject: [PATCH 27/27] change check_crontab messages --- check_crontab.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check_crontab.sh b/check_crontab.sh index caee241..67e6e88 100644 --- a/check_crontab.sh +++ b/check_crontab.sh @@ -5,7 +5,7 @@ target_job="0 0 */2 * * docker run --env-file /home/circleci/.env --network=host crontab_content=$(crontab -l) if [[ $crontab_content == *"$target_job"* ]]; then - echo "Correct pathing" + echo "dbt job has been scheduled" else - echo "Error, incorrect pathing" + echo "error, dbt job has not been properly scheduled" fi