From fe832b5a0b1c4b55eeb577bf71b891bf7d7256ba Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Fri, 1 Mar 2024 15:20:41 -0600 Subject: [PATCH 001/210] Commit submodules --- veda-auth | 1 + veda-backend | 1 + veda-data-airflow | 1 + 3 files changed, 3 insertions(+) create mode 160000 veda-auth create mode 160000 veda-backend create mode 160000 veda-data-airflow diff --git a/veda-auth b/veda-auth new file mode 160000 index 0000000..ff1bb97 --- /dev/null +++ b/veda-auth @@ -0,0 +1 @@ +Subproject commit ff1bb97dac80adac112d2264c34f7d9eead76729 diff --git a/veda-backend b/veda-backend new file mode 160000 index 0000000..0e2bc31 --- /dev/null +++ b/veda-backend @@ -0,0 +1 @@ +Subproject commit 0e2bc3105ba188bf89d5ef274b89dc44b1a20612 diff --git a/veda-data-airflow b/veda-data-airflow new file mode 160000 index 0000000..3d0ab13 --- /dev/null +++ b/veda-data-airflow @@ -0,0 +1 @@ +Subproject commit 3d0ab1363903bfc6cb27a0421dfa7047c636568f From df16d8658c4a308988a2dfbd84155d4724bdad19 Mon Sep 17 00:00:00 2001 From: anayeaye Date: Tue, 26 Mar 2024 12:22:51 -0600 Subject: [PATCH 002/210] fix: invalid url paths in health check --- integration_test/test_api_health.py | 33 +++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/integration_test/test_api_health.py b/integration_test/test_api_health.py index fd44927..5e2e316 100644 --- a/integration_test/test_api_health.py +++ b/integration_test/test_api_health.py @@ -6,16 +6,37 @@ def test_stac_url_returns_200(): - endpoint = os.getenv("VEDA_STAC_URL") - stac_path_prefix = os.getenv("VEDA_STAC_PATH_PREFIX") - url = f"{endpoint.rstrip('/')}/{stac_path_prefix}/_mgmt/ping" + base_url = os.getenv("VEDA_STAC_URL") + stac_root_path = os.getenv("VEDA_STAC_ROOT_PATH") + custom_host = os.getenv("VEDA_CUSTOM_HOST", None) + health_endpoint = "_mgmt/ping" + + url = f"{base_url}{health_endpoint}" + print(f"Checking APIGW stac-api {url=}") response = requests.get(url) assert response.status_code == 200 + if custom_host: + url = f"https://{custom_host}/{stac_root_path.rstrip('/')}/{health_endpoint}" + print(f"Checking custom host stac-api {url=}") + response = requests.get(url) + assert response.status_code == 200 + + def test_raster_url_returns_200(): - endpoint = os.getenv("VEDA_RASTER_URL") - raster_path_prefix = os.getenv("VEDA_RASTER_PATH_PREFIX") - url = f"{endpoint.rstrip('/')}/{raster_path_prefix}/healthz" + base_url = os.getenv("VEDA_RASTER_URL") + raster_root_path = os.getenv("VEDA_RASTER_ROOT_PATH") + custom_host = os.getenv("VEDA_CUSTOM_HOST", None) + health_endpoint = "healthz" + + url = os.path.join(base_url, health_endpoint) + print(f"Checking APIGW raster-api {url=}") response = requests.get(url) assert response.status_code == 200 + + if custom_host: + url = f"https://{custom_host}/{raster_root_path.rstrip('/')}/{health_endpoint}" + print(f"Checking custom host raster-api {url=}") + response = requests.get(url) + assert response.status_code == 200 From 484ac4ff5d38b668cafc3af82f1a759ce4398ad1 Mon Sep 17 00:00:00 2001 From: ividito Date: Tue, 14 May 2024 14:24:12 -0300 Subject: [PATCH 003/210] Add monitoring sub module --- .gitmodules | 3 +++ reuirements.txt => requirements.txt | 0 2 files changed, 3 insertions(+) rename reuirements.txt => requirements.txt (100%) diff --git a/.gitmodules b/.gitmodules index 31265d2..c2c9858 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "veda-backend"] path = veda-backend url = https://github.com/NASA-IMPACT/veda-backend.git +[submodule "veda-monitoring"] + path = veda-monitoring + url = https://github.com/NASA-IMPACT/veda-monitoring.git diff --git a/reuirements.txt b/requirements.txt similarity index 100% rename from reuirements.txt rename to requirements.txt From c9e8468ecfc0e8663ecb4ef9dd6fdedb3ff335a6 Mon Sep 17 00:00:00 2001 From: ividito Date: Tue, 14 May 2024 14:35:34 -0300 Subject: [PATCH 004/210] First pass on monitoring CICD --- .github/workflows/cicd.yml | 43 ++++++++++++++++++++++++++++++++++++++ .gitmodules | 2 +- veda-monitoring | 1 + 3 files changed, 45 insertions(+), 1 deletion(-) create mode 160000 veda-monitoring diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 32190bf..38910a7 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -153,6 +153,49 @@ jobs: backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + deploy-veda-monitoring: + name: deploy VEDA monitoring πŸ‘οΈ + runs-on: ubuntu-latest + env: + DIRECTORY: veda-monitoring + AWS_REGION: "us-west-2" + ENVIRONMENT: ${{ github.event.inputs.environment }} + needs: [deploy-veda-auth, deploy-veda-backend] + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: "true" + submodules: "recursive" + + - name: Checkout to the provided Git Ref + run: | + cd ${{ env.DIRECTORY }} + git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + git fetch --tags origin + echo `git checkout origin/${{ vars.VEDA_MONITORING_GIT_REF || 'main' }}` + git checkout origin/${{ vars.VEDA_MONITORING_GIT_REF || 'main'}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-airflow-deployment" + aws-region: "${{ env.AWS_REGION }}" + + - name: Run deployment + uses: "./veda-monitoring/.github/actions/deploy" + with: + env-file: ".env" + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + dir: "${{ env.DIRECTORY }}" + script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" + auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + + test-deployment: name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ runs-on: ubuntu-latest diff --git a/.gitmodules b/.gitmodules index c2c9858..e1321bf 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,4 +9,4 @@ url = https://github.com/NASA-IMPACT/veda-backend.git [submodule "veda-monitoring"] path = veda-monitoring - url = https://github.com/NASA-IMPACT/veda-monitoring.git + url = git@github.com:NASA-IMPACT/veda-monitoring.git diff --git a/veda-monitoring b/veda-monitoring new file mode 160000 index 0000000..ca38129 --- /dev/null +++ b/veda-monitoring @@ -0,0 +1 @@ +Subproject commit ca381298927ed9dee3b985b01ac2446aa289eedb From f8f495e6ff06f7a8e387c771f17668c43f005734 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Tue, 14 May 2024 14:03:40 -0600 Subject: [PATCH 005/210] monitoring action changes --- .github/workflows/cicd.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 38910a7..bc0f10e 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -182,20 +182,17 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-airflow-deployment" + role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-monitoring-deployment" aws-region: "${{ env.AWS_REGION }}" - name: Run deployment - uses: "./veda-monitoring/.github/actions/deploy" + uses: "./veda-monitoring/.github/actions/cdk-deploy" with: - env-file: ".env" env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} dir: "${{ env.DIRECTORY }}" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" - auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" - + test-deployment: name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ runs-on: ubuntu-latest From 7d4f9c23d8066ef9b8fd23098e8ef28063a7d575 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Tue, 14 May 2024 14:21:58 -0600 Subject: [PATCH 006/210] comment out airflow deployment --- .github/workflows/cicd.yml | 80 +++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index bc0f10e..1e8cc95 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -111,47 +111,47 @@ jobs: outputs: backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} - deploy-veda-data-airflow: - name: deploy VEDA data airflow πŸƒ - runs-on: ubuntu-latest - env: - DIRECTORY: veda-data-airflow - AWS_REGION: "us-west-2" - ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [deploy-veda-auth, deploy-veda-backend] - environment: ${{ github.event.inputs.environment }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - lfs: "true" - submodules: "recursive" - - - name: Checkout to the provided Git Ref - run: | - cd ${{ env.DIRECTORY }} - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch --tags origin - echo `git checkout origin/${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main' }}` - git checkout origin/${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-airflow-deployment" - aws-region: "${{ env.AWS_REGION }}" + # deploy-veda-data-airflow: + # name: deploy VEDA data airflow πŸƒ + # runs-on: ubuntu-latest + # env: + # DIRECTORY: veda-data-airflow + # AWS_REGION: "us-west-2" + # ENVIRONMENT: ${{ github.event.inputs.environment }} + # needs: [deploy-veda-auth, deploy-veda-backend] + # environment: ${{ github.event.inputs.environment }} + + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # lfs: "true" + # submodules: "recursive" + + # - name: Checkout to the provided Git Ref + # run: | + # cd ${{ env.DIRECTORY }} + # git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + # git fetch --tags origin + # echo `git checkout origin/${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main' }}` + # git checkout origin/${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} + + # - name: Configure AWS Credentials + # uses: aws-actions/configure-aws-credentials@v4 + # with: + # role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + # role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-airflow-deployment" + # aws-region: "${{ env.AWS_REGION }}" - - name: Run deployment - uses: "./veda-data-airflow/.github/actions/terraform-deploy" - with: - env-file: ".env" - env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - dir: "${{ env.DIRECTORY }}" - script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" - auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + # - name: Run deployment + # uses: "./veda-data-airflow/.github/actions/terraform-deploy" + # with: + # env-file: ".env" + # env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + # dir: "${{ env.DIRECTORY }}" + # script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + # backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" + # auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" deploy-veda-monitoring: name: deploy VEDA monitoring πŸ‘οΈ From b3eb24fbbf6ec0b91cf782309a338920050d4d99 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Tue, 14 May 2024 14:33:25 -0600 Subject: [PATCH 007/210] fix submodule link --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index e1321bf..c2c9858 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,4 +9,4 @@ url = https://github.com/NASA-IMPACT/veda-backend.git [submodule "veda-monitoring"] path = veda-monitoring - url = git@github.com:NASA-IMPACT/veda-monitoring.git + url = https://github.com/NASA-IMPACT/veda-monitoring.git From f0f452df22f4d8e6de7018b8266d95255339dd4d Mon Sep 17 00:00:00 2001 From: smohiudd Date: Tue, 14 May 2024 15:11:25 -0600 Subject: [PATCH 008/210] add github token --- .github/workflows/cicd.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 1e8cc95..bbf2a6c 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -29,6 +29,7 @@ jobs: with: lfs: "true" submodules: "recursive" + token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout to the provided Git Ref run: | @@ -76,6 +77,7 @@ jobs: with: lfs: "true" submodules: "recursive" + token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout to the provided Git Ref run: | @@ -127,6 +129,7 @@ jobs: # with: # lfs: "true" # submodules: "recursive" + # token: ${{ secrets.GITHUB_TOKEN }} # - name: Checkout to the provided Git Ref # run: | @@ -169,6 +172,7 @@ jobs: with: lfs: "true" submodules: "recursive" + token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout to the provided Git Ref run: | From 6c2cac348095bdee5b97de1759f5873f715443fe Mon Sep 17 00:00:00 2001 From: smohiudd Date: Tue, 14 May 2024 15:47:27 -0600 Subject: [PATCH 009/210] include PAT --- .github/workflows/cicd.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index bbf2a6c..6b6f8f2 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -29,7 +29,7 @@ jobs: with: lfs: "true" submodules: "recursive" - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GH_PAT }} - name: Checkout to the provided Git Ref run: | @@ -77,7 +77,7 @@ jobs: with: lfs: "true" submodules: "recursive" - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GH_PAT }} - name: Checkout to the provided Git Ref run: | @@ -129,7 +129,6 @@ jobs: # with: # lfs: "true" # submodules: "recursive" - # token: ${{ secrets.GITHUB_TOKEN }} # - name: Checkout to the provided Git Ref # run: | @@ -172,7 +171,7 @@ jobs: with: lfs: "true" submodules: "recursive" - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GH_PAT }} - name: Checkout to the provided Git Ref run: | From 41a71546457c80f67f962e60085a42a908bf7b33 Mon Sep 17 00:00:00 2001 From: ividito Date: Wed, 15 May 2024 11:11:16 -0300 Subject: [PATCH 010/210] Modify submodule checkout - this allows us to use fine-grained tokens for specific repos in the future - this should also reduce CICD runtimes --- .github/workflows/cicd.yml | 55 ++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 6b6f8f2..036d3ee 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -24,19 +24,18 @@ jobs: environment: ${{ github.event.inputs.environment }} steps: - - name: Checkout + - name: Checkout deployment repo uses: actions/checkout@v4 with: lfs: "true" - submodules: "recursive" - token: ${{ secrets.GH_PAT }} + submodules: "false" - - name: Checkout to the provided Git Ref - run: | - cd ${{ env.DIRECTORY }} - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch --tags origin - git checkout ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} + - name: Checkout veda-auth "submodule" + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/veda-auth" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 @@ -72,20 +71,19 @@ jobs: environment: ${{ github.event.inputs.environment }} steps: - - name: Checkout + - name: Checkout deployment repo uses: actions/checkout@v4 with: lfs: "true" - submodules: "recursive" - token: ${{ secrets.GH_PAT }} + submodules: "false" + + - name: Checkout veda-backend "submodule" + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main' }} - - name: Checkout to the provided Git Ref - run: | - cd ${{ env.DIRECTORY }} - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch --tags origin - echo `git checkout ${{ vars.VEDA_BACKEND_GIT_REF || 'main' }}` - git checkout ${{ vars.VEDA_BACKEND_GIT_REF || 'main' }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 @@ -95,7 +93,7 @@ jobs: aws-region: "us-west-2" - name: Run deployment - uses: "./veda-backend/.github/actions/cdk-deploy" + uses: "./${{ env.DIRECTORY }}/.github/actions/cdk-deploy" id: deploy_backend_stack with: env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} @@ -170,16 +168,15 @@ jobs: uses: actions/checkout@v4 with: lfs: "true" - submodules: "recursive" - token: ${{ secrets.GH_PAT }} + submodules: "false" - - name: Checkout to the provided Git Ref - run: | - cd ${{ env.DIRECTORY }} - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch --tags origin - echo `git checkout origin/${{ vars.VEDA_MONITORING_GIT_REF || 'main' }}` - git checkout origin/${{ vars.VEDA_MONITORING_GIT_REF || 'main'}} + - name: Checkout veda-monitoring "submodule" + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_MONITORING_GIT_REF || 'main' }} + token: ${{ secrets.GH_PAT }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 From ff1d79413a4915bd631a7dd883368684dc17bea8 Mon Sep 17 00:00:00 2001 From: ividito Date: Wed, 15 May 2024 11:16:35 -0300 Subject: [PATCH 011/210] remove env reference in uses parameter --- .github/workflows/cicd.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 036d3ee..4b70767 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -84,7 +84,6 @@ jobs: path: ${{ env.DIRECTORY }} ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main' }} - - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 with: @@ -93,7 +92,7 @@ jobs: aws-region: "us-west-2" - name: Run deployment - uses: "./${{ env.DIRECTORY }}/.github/actions/cdk-deploy" + uses: "./veda-backend/.github/actions/cdk-deploy" id: deploy_backend_stack with: env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} From 5255471c42d7a4fc8ff091a92dc427ce2ba1d24c Mon Sep 17 00:00:00 2001 From: Sanjog Thapa Date: Mon, 3 Jun 2024 10:53:17 -0500 Subject: [PATCH 012/210] shorten the role session name for aws action char limit, still making it comprehensible. --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 32190bf..0622bab 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -140,7 +140,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-airflow-deployment" + role-session-name: "gh-${{ env.ENVIRONMENT }}-airflow-deployment" aws-region: "${{ env.AWS_REGION }}" - name: Run deployment From c6a1a10ef6b45c53224ab28e623c151e203e2c5b Mon Sep 17 00:00:00 2001 From: Sanjog Thapa Date: Mon, 3 Jun 2024 11:07:38 -0500 Subject: [PATCH 013/210] shorten the role session name on auth, backend and test deployment workflows. --- .github/workflows/cicd.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 0622bab..b2f712e 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -41,7 +41,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-auth-deployment" + role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" aws-region: "us-west-2" - name: Run auth deployment @@ -89,7 +89,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-backend-deployment" + role-session-name: "gh-${{ env.ENVIRONMENT }}-backend-deployment" aws-region: "us-west-2" - name: Run deployment @@ -177,7 +177,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-integration-test" + role-session-name: "gh-${{ env.ENVIRONMENT }}-integration-test" aws-region: "${{ env.AWS_DEFAULT_REGION }}" - name: Install python dependencies From ae8ee1d9b17d82dc35b223e97ee06c201770ab04 Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Mon, 3 Jun 2024 11:59:09 -0700 Subject: [PATCH 014/210] feat: update cicd.yml to support checkout of tags for veda-data-airflow --- .github/workflows/cicd.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 32190bf..e23b0e2 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -133,8 +133,8 @@ jobs: cd ${{ env.DIRECTORY }} git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" git fetch --tags origin - echo `git checkout origin/${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main' }}` - git checkout origin/${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} + echo `git checkout ${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main' }}` + git checkout ${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -142,7 +142,7 @@ jobs: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-airflow-deployment" aws-region: "${{ env.AWS_REGION }}" - + - name: Run deployment uses: "./veda-data-airflow/.github/actions/terraform-deploy" with: From 0f7dec550999ae8fd5bbea0fc9441dd1c7f79117 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Wed, 12 Jun 2024 17:45:45 +0300 Subject: [PATCH 015/210] aws credentials v2 --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 4b70767..c791480 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -178,7 +178,7 @@ jobs: token: ${{ secrets.GH_PAT }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-monitoring-deployment" From 9cb1a1c3f43a34426bd22e36a5ceb5e44f2c2a90 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 17 Jun 2024 07:35:43 -0600 Subject: [PATCH 016/210] assume role duration --- .github/workflows/cicd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index c791480..b7b9f6e 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -181,6 +181,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-duration-seconds: 120 role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-monitoring-deployment" aws-region: "${{ env.AWS_REGION }}" From f3fa24215f7ba46a38d830dd35320cbb0271cb54 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 17 Jun 2024 08:41:30 -0600 Subject: [PATCH 017/210] use github token --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index b7b9f6e..831774d 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -175,7 +175,7 @@ jobs: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} ref: ${{ vars.VEDA_MONITORING_GIT_REF || 'main' }} - token: ${{ secrets.GH_PAT }} + token: ${{ secrets.GITHUB_TOKEN }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 From 5f66cbca01f9d2671ac647cb1c3dbf0a1c187fa5 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 17 Jun 2024 08:48:24 -0600 Subject: [PATCH 018/210] use github token --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 831774d..b6cd026 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -168,6 +168,7 @@ jobs: with: lfs: "true" submodules: "false" + token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout veda-monitoring "submodule" uses: actions/checkout@v4 @@ -175,7 +176,6 @@ jobs: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} ref: ${{ vars.VEDA_MONITORING_GIT_REF || 'main' }} - token: ${{ secrets.GITHUB_TOKEN }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 From c9a2000fc94b22d38cf861d458e3186883cb4dca Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 17 Jun 2024 08:51:31 -0600 Subject: [PATCH 019/210] add token to submodule checkout --- .github/workflows/cicd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index b6cd026..9c3f3b8 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -176,6 +176,7 @@ jobs: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} ref: ${{ vars.VEDA_MONITORING_GIT_REF || 'main' }} + token: ${{ secrets.GITHUB_TOKEN }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 From 8ee1a6a405822e981002790635edbd40955f8bfc Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 17 Jun 2024 12:17:37 -0600 Subject: [PATCH 020/210] add PAT for veda monitoring --- .github/workflows/cicd.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 9c3f3b8..b7b9f6e 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -168,7 +168,6 @@ jobs: with: lfs: "true" submodules: "false" - token: ${{ secrets.GITHUB_TOKEN }} - name: Checkout veda-monitoring "submodule" uses: actions/checkout@v4 @@ -176,7 +175,7 @@ jobs: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} ref: ${{ vars.VEDA_MONITORING_GIT_REF || 'main' }} - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GH_PAT }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 From 42115970e85189b6a7ef096ec3172f19f8e2e2d1 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 17 Jun 2024 16:00:31 -0600 Subject: [PATCH 021/210] uncomment --- .github/workflows/cicd.yml | 80 +++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index b7b9f6e..685fb85 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -110,47 +110,47 @@ jobs: outputs: backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} - # deploy-veda-data-airflow: - # name: deploy VEDA data airflow πŸƒ - # runs-on: ubuntu-latest - # env: - # DIRECTORY: veda-data-airflow - # AWS_REGION: "us-west-2" - # ENVIRONMENT: ${{ github.event.inputs.environment }} - # needs: [deploy-veda-auth, deploy-veda-backend] - # environment: ${{ github.event.inputs.environment }} - - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # lfs: "true" - # submodules: "recursive" - - # - name: Checkout to the provided Git Ref - # run: | - # cd ${{ env.DIRECTORY }} - # git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - # git fetch --tags origin - # echo `git checkout origin/${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main' }}` - # git checkout origin/${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} - - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - # role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-airflow-deployment" - # aws-region: "${{ env.AWS_REGION }}" + deploy-veda-data-airflow: + name: deploy VEDA data airflow πŸƒ + runs-on: ubuntu-latest + env: + DIRECTORY: veda-data-airflow + AWS_REGION: "us-west-2" + ENVIRONMENT: ${{ github.event.inputs.environment }} + needs: [deploy-veda-auth, deploy-veda-backend] + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: "true" + submodules: "recursive" + + - name: Checkout to the provided Git Ref + run: | + cd ${{ env.DIRECTORY }} + git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + git fetch --tags origin + echo `git checkout origin/${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main' }}` + git checkout origin/${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-airflow-deployment" + aws-region: "${{ env.AWS_REGION }}" - # - name: Run deployment - # uses: "./veda-data-airflow/.github/actions/terraform-deploy" - # with: - # env-file: ".env" - # env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - # dir: "${{ env.DIRECTORY }}" - # script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - # backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" - # auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + - name: Run deployment + uses: "./veda-data-airflow/.github/actions/terraform-deploy" + with: + env-file: ".env" + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + dir: "${{ env.DIRECTORY }}" + script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" + auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" deploy-veda-monitoring: name: deploy VEDA monitoring πŸ‘οΈ From c034479c3e36b19306bd3976dda4cb92723c4a83 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 17 Jun 2024 16:05:25 -0600 Subject: [PATCH 022/210] revert cicd changes backend, auth --- .github/workflows/cicd.yml | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 685fb85..60d9bd5 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -24,24 +24,24 @@ jobs: environment: ${{ github.event.inputs.environment }} steps: - - name: Checkout deployment repo + - name: Checkout uses: actions/checkout@v4 with: lfs: "true" - submodules: "false" + submodules: "recursive" - - name: Checkout veda-auth "submodule" - uses: actions/checkout@v4 - with: - repository: "NASA-IMPACT/veda-auth" - path: ${{ env.DIRECTORY }} - ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} + - name: Checkout to the provided Git Ref + run: | + cd ${{ env.DIRECTORY }} + git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + git fetch --tags origin + git checkout ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-auth-deployment" + role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" aws-region: "us-west-2" - name: Run auth deployment @@ -71,24 +71,25 @@ jobs: environment: ${{ github.event.inputs.environment }} steps: - - name: Checkout deployment repo + - name: Checkout uses: actions/checkout@v4 with: lfs: "true" - submodules: "false" + submodules: "recursive" - - name: Checkout veda-backend "submodule" - uses: actions/checkout@v4 - with: - repository: "NASA-IMPACT/${{ env.DIRECTORY }}" - path: ${{ env.DIRECTORY }} - ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main' }} + - name: Checkout to the provided Git Ref + run: | + cd ${{ env.DIRECTORY }} + git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + git fetch --tags origin + echo `git checkout ${{ vars.VEDA_BACKEND_GIT_REF || 'main' }}` + git checkout ${{ vars.VEDA_BACKEND_GIT_REF || 'main' }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-backend-deployment" + role-session-name: "gh-${{ env.ENVIRONMENT }}-backend-deployment" aws-region: "us-west-2" - name: Run deployment From bb91ec9f8250e3d46e22f8853216c92dc03cb52f Mon Sep 17 00:00:00 2001 From: smohiudd Date: Mon, 17 Jun 2024 16:10:49 -0600 Subject: [PATCH 023/210] remove assume role duration --- .github/workflows/cicd.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 2a16052..e42d3c6 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -182,7 +182,6 @@ jobs: uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-duration-seconds: 120 role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-monitoring-deployment" aws-region: "${{ env.AWS_REGION }}" From 7e8fb33c23a0a0e1680ebe56e92bfa7c334f0f7f Mon Sep 17 00:00:00 2001 From: smohiudd Date: Wed, 19 Jun 2024 15:16:44 -0600 Subject: [PATCH 024/210] change submodule checkout --- .github/workflows/cicd.yml | 46 ++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index e42d3c6..7d88f26 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -28,14 +28,14 @@ jobs: uses: actions/checkout@v4 with: lfs: "true" - submodules: "recursive" + submodules: "false" - - name: Checkout to the provided Git Ref - run: | - cd ${{ env.DIRECTORY }} - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch --tags origin - git checkout ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} + - name: Checkout veda-auth submodule + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 @@ -75,15 +75,14 @@ jobs: uses: actions/checkout@v4 with: lfs: "true" - submodules: "recursive" + submodules: "false" - - name: Checkout to the provided Git Ref - run: | - cd ${{ env.DIRECTORY }} - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch --tags origin - echo `git checkout ${{ vars.VEDA_BACKEND_GIT_REF || 'main' }}` - git checkout ${{ vars.VEDA_BACKEND_GIT_REF || 'main' }} + - name: Checkout veda-backend submodule + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main'}} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 @@ -126,15 +125,14 @@ jobs: uses: actions/checkout@v4 with: lfs: "true" - submodules: "recursive" + submodules: "false" - - name: Checkout to the provided Git Ref - run: | - cd ${{ env.DIRECTORY }} - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch --tags origin - echo `git checkout ${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main' }}` - git checkout ${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} + - name: Checkout veda-data-airflow submodule + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -182,7 +180,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "${{ env.DIRECTORY }}-github-${{ env.ENVIRONMENT }}-monitoring-deployment" + role-session-name: "gh-${{ env.ENVIRONMENT }}-monitoring-deployment" aws-region: "${{ env.AWS_REGION }}" - name: Run deployment From c55a54dd69e1c9115cf0373b38718755f080ae64 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Thu, 20 Jun 2024 10:07:30 -0600 Subject: [PATCH 025/210] veda monitoring conditional --- .github/workflows/cicd.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 7d88f26..cee40fa 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -158,10 +158,12 @@ jobs: DIRECTORY: veda-monitoring AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} + GH_PAT_CHECK: ${{ secrets.GH_PAT }} needs: [deploy-veda-auth, deploy-veda-backend] environment: ${{ github.event.inputs.environment }} steps: + - if: ${{ env.GH_PAT_CHECK != '' }} - name: Checkout uses: actions/checkout@v4 with: From 415dac84ef06669069c0b85c8d46b25cf96ea55c Mon Sep 17 00:00:00 2001 From: smohiudd Date: Thu, 20 Jun 2024 10:14:08 -0600 Subject: [PATCH 026/210] move conditional --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index cee40fa..6fb8dd5 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -160,10 +160,10 @@ jobs: ENVIRONMENT: ${{ github.event.inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} needs: [deploy-veda-auth, deploy-veda-backend] + if: ${{ env.GH_PAT_CHECK != '' }} environment: ${{ github.event.inputs.environment }} steps: - - if: ${{ env.GH_PAT_CHECK != '' }} - name: Checkout uses: actions/checkout@v4 with: From e78e40ad7f389ab03d4d7e8e716fbc653d92fca4 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Thu, 20 Jun 2024 10:17:02 -0600 Subject: [PATCH 027/210] move conditional --- .github/workflows/cicd.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 6fb8dd5..4f61b93 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -160,17 +160,18 @@ jobs: ENVIRONMENT: ${{ github.event.inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} needs: [deploy-veda-auth, deploy-veda-backend] - if: ${{ env.GH_PAT_CHECK != '' }} environment: ${{ github.event.inputs.environment }} steps: - name: Checkout + if: ${{ env.GH_PAT_CHECK != '' }} uses: actions/checkout@v4 with: lfs: "true" submodules: "false" - name: Checkout veda-monitoring "submodule" + if: ${{ env.GH_PAT_CHECK != '' }} uses: actions/checkout@v4 with: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" @@ -179,6 +180,7 @@ jobs: token: ${{ secrets.GH_PAT }} - name: Configure AWS Credentials + if: ${{ env.GH_PAT_CHECK != '' }} uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} @@ -186,6 +188,7 @@ jobs: aws-region: "${{ env.AWS_REGION }}" - name: Run deployment + if: ${{ env.GH_PAT_CHECK != '' }} uses: "./veda-monitoring/.github/actions/cdk-deploy" with: env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} From 43e31b0efee209ce5ce537e928c901fb11bbe3c0 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Thu, 20 Jun 2024 10:38:00 -0600 Subject: [PATCH 028/210] fix airflow ref --- .github/workflows/cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 4f61b93..73a5848 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -132,7 +132,7 @@ jobs: with: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} - ref: ${{ vars.vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} + ref: ${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -159,7 +159,7 @@ jobs: AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} - needs: [deploy-veda-auth, deploy-veda-backend] + needs: [deploy-veda-auth, deploy-veda-backend, deploy-veda-data-airflow] environment: ${{ github.event.inputs.environment }} steps: From ca70c83dccfeebd5fa551c36e4eddc45701ead29 Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Tue, 30 Jul 2024 16:01:11 -0700 Subject: [PATCH 029/210] feat: add step to get workflows api endpoint --- .github/workflows/cicd.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 73a5848..ee7d672 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -151,6 +151,13 @@ jobs: backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + - name: Get Workflows API Endpoint + id: get_workflows_api_endpoint + shell: bash + run: | + workflows_api_value=$(jq -r '.workflows_api.value' ${HOME}/terraform_outputs.json) + echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT + deploy-veda-monitoring: name: deploy VEDA monitoring πŸ‘οΈ runs-on: ubuntu-latest @@ -186,7 +193,7 @@ jobs: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-monitoring-deployment" aws-region: "${{ env.AWS_REGION }}" - + - name: Run deployment if: ${{ env.GH_PAT_CHECK != '' }} uses: "./veda-monitoring/.github/actions/cdk-deploy" From ffb270a199287417ceb2264e7bd11d7ae6e83264 Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Tue, 30 Jul 2024 17:17:15 -0700 Subject: [PATCH 030/210] fix: cat terraform outputs file --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index ee7d672..f5b9640 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -155,7 +155,7 @@ jobs: id: get_workflows_api_endpoint shell: bash run: | - workflows_api_value=$(jq -r '.workflows_api.value' ${HOME}/terraform_outputs.json) + workflows_api_value=$(cat ${HOME}/terraform_outputs.json) echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT deploy-veda-monitoring: From 516a666b8cf111f152d82919648c9b6ee2e1e544 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Wed, 31 Jul 2024 11:46:29 -0500 Subject: [PATCH 031/210] Add veda-features-api-cdk submodule --- .gitmodules | 3 +++ veda-features-api-cdk | 1 + 2 files changed, 4 insertions(+) create mode 160000 veda-features-api-cdk diff --git a/.gitmodules b/.gitmodules index c2c9858..7e3e264 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "veda-monitoring"] path = veda-monitoring url = https://github.com/NASA-IMPACT/veda-monitoring.git +[submodule "veda-features-api-cdk"] + path = veda-features-api-cdk + url = git@github.com:NASA-IMPACT/veda-features-api-cdk.git diff --git a/veda-features-api-cdk b/veda-features-api-cdk new file mode 160000 index 0000000..8e32f0f --- /dev/null +++ b/veda-features-api-cdk @@ -0,0 +1 @@ +Subproject commit 8e32f0faecddb8d4d00e6b9b767132f5e8d286d3 From 0985d58eaee1deec1ab823bc7d62e5960d7f30cc Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Wed, 31 Jul 2024 11:46:56 -0500 Subject: [PATCH 032/210] WIP deploy features api step in cicd.yml --- .github/workflows/cicd.yml | 49 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 73a5848..1e30647 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -151,9 +151,58 @@ jobs: backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + deploy-veda-features-api: + name: Deploy VEDA features-api βš™οΈ + runs-on: ubuntu-latest + if: vars.DEPLOY_FEATURES_API + env: + DIRECTORY: veda-features-api-cdk + ENVIRONMENT: ${{ github.event.inputs.environment }} + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: "true" + submodules: "false" + + - name: Checkout veda-features-api submodule + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_FEATURES_API_GIT_REF || 'main'}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-features-deployment" + aws-region: "us-west-2" + + - name: Run deployment + uses: "./veda-features-api-cdk/.github/actions/cdk-deploy" + id: deploy_features_stack + with: + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + dir: "${{ env.DIRECTORY }}" + script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + + - name: Get Features API Stack Name + id: get_features_api_stack + shell: bash + run: | + stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) + echo "features_api_stackname=$stack" >> $GITHUB_OUTPUT + + outputs: + features_api_stack_name: ${{ steps.get_features_api_stack.outputs.features_api_stackname }} + deploy-veda-monitoring: name: deploy VEDA monitoring πŸ‘οΈ runs-on: ubuntu-latest + if: vars.DEPLOY_MONITORING env: DIRECTORY: veda-monitoring AWS_REGION: "us-west-2" From b02ce5493a06c7fd2906e1431fcb531eb71937b1 Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Wed, 31 Jul 2024 10:01:25 -0700 Subject: [PATCH 033/210] fix: revert to extract workflows api value --- .github/workflows/cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index f5b9640..f6f50bc 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -155,8 +155,8 @@ jobs: id: get_workflows_api_endpoint shell: bash run: | - workflows_api_value=$(cat ${HOME}/terraform_outputs.json) - echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT + workflows_api_value=$(jq -r '.workflows_api.value' ${HOME}/terraform_outputs.json) + echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT deploy-veda-monitoring: name: deploy VEDA monitoring πŸ‘οΈ From 7e3d3101d4fc8c2d1e2744975e3dd8c20e6848f3 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Wed, 31 Jul 2024 13:38:46 -0500 Subject: [PATCH 034/210] Test removing submodule --- .gitmodules | 3 --- veda-features-api-cdk | 1 - 2 files changed, 4 deletions(-) delete mode 160000 veda-features-api-cdk diff --git a/.gitmodules b/.gitmodules index 7e3e264..c2c9858 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,6 +10,3 @@ [submodule "veda-monitoring"] path = veda-monitoring url = https://github.com/NASA-IMPACT/veda-monitoring.git -[submodule "veda-features-api-cdk"] - path = veda-features-api-cdk - url = git@github.com:NASA-IMPACT/veda-features-api-cdk.git diff --git a/veda-features-api-cdk b/veda-features-api-cdk deleted file mode 160000 index 8e32f0f..0000000 --- a/veda-features-api-cdk +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8e32f0faecddb8d4d00e6b9b767132f5e8d286d3 From 2815a1d7e955e6131a9d2b4a31faaf7539357a44 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Wed, 31 Jul 2024 13:59:13 -0500 Subject: [PATCH 035/210] Update emoji for features api --- .github/workflows/cicd.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 1e30647..113c7fe 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -152,7 +152,7 @@ jobs: auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" deploy-veda-features-api: - name: Deploy VEDA features-api βš™οΈ + name: Deploy VEDA features-api πŸ—ΊοΈ runs-on: ubuntu-latest if: vars.DEPLOY_FEATURES_API env: @@ -244,7 +244,6 @@ jobs: dir: "${{ env.DIRECTORY }}" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - test-deployment: name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ runs-on: ubuntu-latest From eb3cda05f8c339f93f314b23822325adf96be9ec Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Wed, 31 Jul 2024 14:18:56 -0500 Subject: [PATCH 036/210] Add concurrency --- .github/workflows/cicd.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 113c7fe..30635e1 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -12,6 +12,10 @@ on: required: true description: Environment to deploy to +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.environment }} + cancel-in-progress: true + run-name: Deploy to ${{ inputs.environment }} environment by @${{ github.actor }} jobs: @@ -154,7 +158,7 @@ jobs: deploy-veda-features-api: name: Deploy VEDA features-api πŸ—ΊοΈ runs-on: ubuntu-latest - if: vars.DEPLOY_FEATURES_API + if: ${{ vars.DEPLOY_FEATURES_API == true }} env: DIRECTORY: veda-features-api-cdk ENVIRONMENT: ${{ github.event.inputs.environment }} From 65888fda72844c41e8a2c3d5c9c7b0b5ecc3ef8d Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Wed, 31 Jul 2024 12:52:02 -0700 Subject: [PATCH 037/210] fix: test new changes --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index f6f50bc..b746a68 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -155,7 +155,7 @@ jobs: id: get_workflows_api_endpoint shell: bash run: | - workflows_api_value=$(jq -r '.workflows_api.value' ${HOME}/terraform_outputs.json) + workflows_api_value=$(cat ${HOME}/terraform_outputs.json) echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT deploy-veda-monitoring: From 188a9b0b3f89fcf5da40ad7ea244e8c9fbe9a5bf Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Wed, 31 Jul 2024 13:19:50 -0700 Subject: [PATCH 038/210] fix: testing omitting jobs for faster testing --- .github/workflows/cicd.yml | 278 +++++++++++++++++++------------------ 1 file changed, 141 insertions(+), 137 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index b746a68..ba4e938 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -15,100 +15,100 @@ on: run-name: Deploy to ${{ inputs.environment }} environment by @${{ github.actor }} jobs: - deploy-veda-auth: - name: Deploy VEDA auth πŸ” - runs-on: ubuntu-latest - env: - DIRECTORY: veda-auth - ENVIRONMENT: ${{ github.event.inputs.environment }} - environment: ${{ github.event.inputs.environment }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - lfs: "true" - submodules: "false" - - - name: Checkout veda-auth submodule - uses: actions/checkout@v4 - with: - repository: "NASA-IMPACT/${{ env.DIRECTORY }}" - path: ${{ env.DIRECTORY }} - ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" - aws-region: "us-west-2" - - - name: Run auth deployment - id: deploy_auth_stack - uses: "./veda-auth/.github/actions/cdk-deploy" - with: - dir: "${{ env.DIRECTORY }}" - env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - - - name: Get Auth Stack Name - id: get_auth_stack - shell: bash - run: | - stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) - echo "auth_stackname=$stack" >> $GITHUB_OUTPUT - - outputs: - auth_stack_name: ${{ steps.get_auth_stack.outputs.auth_stackname }} - - deploy-veda-backend: - name: Deploy VEDA backend βš™οΈ - runs-on: ubuntu-latest - env: - DIRECTORY: veda-backend - ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [deploy-veda-auth] - environment: ${{ github.event.inputs.environment }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - lfs: "true" - submodules: "false" - - - name: Checkout veda-backend submodule - uses: actions/checkout@v4 - with: - repository: "NASA-IMPACT/${{ env.DIRECTORY }}" - path: ${{ env.DIRECTORY }} - ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main'}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "gh-${{ env.ENVIRONMENT }}-backend-deployment" - aws-region: "us-west-2" - - - name: Run deployment - uses: "./veda-backend/.github/actions/cdk-deploy" - id: deploy_backend_stack - with: - env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" - dir: "${{ env.DIRECTORY }}" - script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - - - name: Get Backend Stack Name - id: get_backend_stack - shell: bash - run: | - stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) - echo "backend_stackname=$stack" >> $GITHUB_OUTPUT - - outputs: - backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} + # deploy-veda-auth: + # name: Deploy VEDA auth πŸ” + # runs-on: ubuntu-latest + # env: + # DIRECTORY: veda-auth + # ENVIRONMENT: ${{ github.event.inputs.environment }} + # environment: ${{ github.event.inputs.environment }} + + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # lfs: "true" + # submodules: "false" + + # - name: Checkout veda-auth submodule + # uses: actions/checkout@v4 + # with: + # repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + # path: ${{ env.DIRECTORY }} + # ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} + + # - name: Configure AWS Credentials + # uses: aws-actions/configure-aws-credentials@v2 + # with: + # role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + # role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" + # aws-region: "us-west-2" + + # - name: Run auth deployment + # id: deploy_auth_stack + # uses: "./veda-auth/.github/actions/cdk-deploy" + # with: + # dir: "${{ env.DIRECTORY }}" + # env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + + # - name: Get Auth Stack Name + # id: get_auth_stack + # shell: bash + # run: | + # stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) + # echo "auth_stackname=$stack" >> $GITHUB_OUTPUT + + # outputs: + # auth_stack_name: ${{ steps.get_auth_stack.outputs.auth_stackname }} + + # deploy-veda-backend: + # name: Deploy VEDA backend βš™οΈ + # runs-on: ubuntu-latest + # env: + # DIRECTORY: veda-backend + # ENVIRONMENT: ${{ github.event.inputs.environment }} + # needs: [deploy-veda-auth] + # environment: ${{ github.event.inputs.environment }} + + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # with: + # lfs: "true" + # submodules: "false" + + # - name: Checkout veda-backend submodule + # uses: actions/checkout@v4 + # with: + # repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + # path: ${{ env.DIRECTORY }} + # ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main'}} + + # - name: Configure AWS Credentials + # uses: aws-actions/configure-aws-credentials@v2 + # with: + # role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + # role-session-name: "gh-${{ env.ENVIRONMENT }}-backend-deployment" + # aws-region: "us-west-2" + + # - name: Run deployment + # uses: "./veda-backend/.github/actions/cdk-deploy" + # id: deploy_backend_stack + # with: + # env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + # auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + # dir: "${{ env.DIRECTORY }}" + # script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + + # - name: Get Backend Stack Name + # id: get_backend_stack + # shell: bash + # run: | + # stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) + # echo "backend_stackname=$stack" >> $GITHUB_OUTPUT + + # outputs: + # backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} deploy-veda-data-airflow: name: deploy VEDA data airflow πŸƒ @@ -117,7 +117,7 @@ jobs: DIRECTORY: veda-data-airflow AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [deploy-veda-auth, deploy-veda-backend] + # needs: [deploy-veda-auth, deploy-veda-backend] environment: ${{ github.event.inputs.environment }} steps: @@ -148,8 +148,10 @@ jobs: env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} dir: "${{ env.DIRECTORY }}" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" - auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + # backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" + backend_stack_name: "veda-backend-test" + # auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + auth_stack_name: "veda-auth-stack-test" - name: Get Workflows API Endpoint id: get_workflows_api_endpoint @@ -157,6 +159,7 @@ jobs: run: | workflows_api_value=$(cat ${HOME}/terraform_outputs.json) echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT + cat $GITHUB_OUTPUT deploy-veda-monitoring: name: deploy VEDA monitoring πŸ‘οΈ @@ -166,7 +169,8 @@ jobs: AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} - needs: [deploy-veda-auth, deploy-veda-backend, deploy-veda-data-airflow] + # needs: [deploy-veda-auth, deploy-veda-backend, deploy-veda-data-airflow] + needs: [deploy-veda-data-airflow] environment: ${{ github.event.inputs.environment }} steps: @@ -203,42 +207,42 @@ jobs: script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - test-deployment: - name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ - runs-on: ubuntu-latest - needs: [ deploy-veda-backend ] - env: - DIRECTORY: integration_test - ENVIRONMENT: ${{ github.event.inputs.environment }} - AWS_DEFAULT_REGION: us-west-2 - environment: ${{ github.event.inputs.environment }} - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - cache-dependency-path: | - ${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "gh-${{ env.ENVIRONMENT }}-integration-test" - aws-region: "${{ env.AWS_DEFAULT_REGION }}" - - - name: Install python dependencies - shell: bash - working-directory: ${{ env.DIRECTORY }} - run: | - pip install -r requirements.txt - python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" - - - name: Integration test - shell: bash - working-directory: ${{ env.DIRECTORY }} - run: | - pytest . + # test-deployment: + # name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ + # runs-on: ubuntu-latest + # needs: [ deploy-veda-backend ] + # env: + # DIRECTORY: integration_test + # ENVIRONMENT: ${{ github.event.inputs.environment }} + # AWS_DEFAULT_REGION: us-west-2 + # environment: ${{ github.event.inputs.environment }} + # steps: + # - uses: actions/checkout@v2 + + # - name: Set up Python + # uses: actions/setup-python@v4 + # with: + # python-version: "3.10" + # cache: "pip" + # cache-dependency-path: | + # ${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt + + # - name: Configure AWS Credentials + # uses: aws-actions/configure-aws-credentials@v2 + # with: + # role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + # role-session-name: "gh-${{ env.ENVIRONMENT }}-integration-test" + # aws-region: "${{ env.AWS_DEFAULT_REGION }}" + + # - name: Install python dependencies + # shell: bash + # working-directory: ${{ env.DIRECTORY }} + # run: | + # pip install -r requirements.txt + # python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" + + # - name: Integration test + # shell: bash + # working-directory: ${{ env.DIRECTORY }} + # run: | + # pytest . From 3d1667162417c5d74fc0352cff1c25f07f5453de Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Wed, 31 Jul 2024 13:51:44 -0700 Subject: [PATCH 039/210] fix: undo commented out jobs --- .github/workflows/cicd.yml | 277 ++++++++++++++++++------------------- 1 file changed, 137 insertions(+), 140 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index ba4e938..8122d9e 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -15,100 +15,100 @@ on: run-name: Deploy to ${{ inputs.environment }} environment by @${{ github.actor }} jobs: - # deploy-veda-auth: - # name: Deploy VEDA auth πŸ” - # runs-on: ubuntu-latest - # env: - # DIRECTORY: veda-auth - # ENVIRONMENT: ${{ github.event.inputs.environment }} - # environment: ${{ github.event.inputs.environment }} - - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # lfs: "true" - # submodules: "false" - - # - name: Checkout veda-auth submodule - # uses: actions/checkout@v4 - # with: - # repository: "NASA-IMPACT/${{ env.DIRECTORY }}" - # path: ${{ env.DIRECTORY }} - # ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} - - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - # role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" - # aws-region: "us-west-2" - - # - name: Run auth deployment - # id: deploy_auth_stack - # uses: "./veda-auth/.github/actions/cdk-deploy" - # with: - # dir: "${{ env.DIRECTORY }}" - # env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - - # - name: Get Auth Stack Name - # id: get_auth_stack - # shell: bash - # run: | - # stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) - # echo "auth_stackname=$stack" >> $GITHUB_OUTPUT - - # outputs: - # auth_stack_name: ${{ steps.get_auth_stack.outputs.auth_stackname }} - - # deploy-veda-backend: - # name: Deploy VEDA backend βš™οΈ - # runs-on: ubuntu-latest - # env: - # DIRECTORY: veda-backend - # ENVIRONMENT: ${{ github.event.inputs.environment }} - # needs: [deploy-veda-auth] - # environment: ${{ github.event.inputs.environment }} - - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # lfs: "true" - # submodules: "false" - - # - name: Checkout veda-backend submodule - # uses: actions/checkout@v4 - # with: - # repository: "NASA-IMPACT/${{ env.DIRECTORY }}" - # path: ${{ env.DIRECTORY }} - # ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main'}} - - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - # role-session-name: "gh-${{ env.ENVIRONMENT }}-backend-deployment" - # aws-region: "us-west-2" - - # - name: Run deployment - # uses: "./veda-backend/.github/actions/cdk-deploy" - # id: deploy_backend_stack - # with: - # env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - # auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" - # dir: "${{ env.DIRECTORY }}" - # script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - - # - name: Get Backend Stack Name - # id: get_backend_stack - # shell: bash - # run: | - # stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) - # echo "backend_stackname=$stack" >> $GITHUB_OUTPUT - - # outputs: - # backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} + deploy-veda-auth: + name: Deploy VEDA auth πŸ” + runs-on: ubuntu-latest + env: + DIRECTORY: veda-auth + ENVIRONMENT: ${{ github.event.inputs.environment }} + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: "true" + submodules: "false" + + - name: Checkout veda-auth submodule + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" + aws-region: "us-west-2" + + - name: Run auth deployment + id: deploy_auth_stack + uses: "./veda-auth/.github/actions/cdk-deploy" + with: + dir: "${{ env.DIRECTORY }}" + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + + - name: Get Auth Stack Name + id: get_auth_stack + shell: bash + run: | + stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) + echo "auth_stackname=$stack" >> $GITHUB_OUTPUT + + outputs: + auth_stack_name: ${{ steps.get_auth_stack.outputs.auth_stackname }} + + deploy-veda-backend: + name: Deploy VEDA backend βš™οΈ + runs-on: ubuntu-latest + env: + DIRECTORY: veda-backend + ENVIRONMENT: ${{ github.event.inputs.environment }} + needs: [deploy-veda-auth] + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: "true" + submodules: "false" + + - name: Checkout veda-backend submodule + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main'}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-backend-deployment" + aws-region: "us-west-2" + + - name: Run deployment + uses: "./veda-backend/.github/actions/cdk-deploy" + id: deploy_backend_stack + with: + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + dir: "${{ env.DIRECTORY }}" + script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + + - name: Get Backend Stack Name + id: get_backend_stack + shell: bash + run: | + stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) + echo "backend_stackname=$stack" >> $GITHUB_OUTPUT + + outputs: + backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} deploy-veda-data-airflow: name: deploy VEDA data airflow πŸƒ @@ -117,7 +117,7 @@ jobs: DIRECTORY: veda-data-airflow AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} - # needs: [deploy-veda-auth, deploy-veda-backend] + needs: [deploy-veda-auth, deploy-veda-backend] environment: ${{ github.event.inputs.environment }} steps: @@ -148,10 +148,8 @@ jobs: env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} dir: "${{ env.DIRECTORY }}" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - # backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" - backend_stack_name: "veda-backend-test" - # auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" - auth_stack_name: "veda-auth-stack-test" + backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" + auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" - name: Get Workflows API Endpoint id: get_workflows_api_endpoint @@ -169,8 +167,7 @@ jobs: AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} - # needs: [deploy-veda-auth, deploy-veda-backend, deploy-veda-data-airflow] - needs: [deploy-veda-data-airflow] + needs: [deploy-veda-auth, deploy-veda-backend, deploy-veda-data-airflow] environment: ${{ github.event.inputs.environment }} steps: @@ -207,42 +204,42 @@ jobs: script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - # test-deployment: - # name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ - # runs-on: ubuntu-latest - # needs: [ deploy-veda-backend ] - # env: - # DIRECTORY: integration_test - # ENVIRONMENT: ${{ github.event.inputs.environment }} - # AWS_DEFAULT_REGION: us-west-2 - # environment: ${{ github.event.inputs.environment }} - # steps: - # - uses: actions/checkout@v2 - - # - name: Set up Python - # uses: actions/setup-python@v4 - # with: - # python-version: "3.10" - # cache: "pip" - # cache-dependency-path: | - # ${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt - - # - name: Configure AWS Credentials - # uses: aws-actions/configure-aws-credentials@v2 - # with: - # role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - # role-session-name: "gh-${{ env.ENVIRONMENT }}-integration-test" - # aws-region: "${{ env.AWS_DEFAULT_REGION }}" - - # - name: Install python dependencies - # shell: bash - # working-directory: ${{ env.DIRECTORY }} - # run: | - # pip install -r requirements.txt - # python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" - - # - name: Integration test - # shell: bash - # working-directory: ${{ env.DIRECTORY }} - # run: | - # pytest . + test-deployment: + name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ + runs-on: ubuntu-latest + needs: [ deploy-veda-backend ] + env: + DIRECTORY: integration_test + ENVIRONMENT: ${{ github.event.inputs.environment }} + AWS_DEFAULT_REGION: us-west-2 + environment: ${{ github.event.inputs.environment }} + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + cache-dependency-path: | + ${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-integration-test" + aws-region: "${{ env.AWS_DEFAULT_REGION }}" + + - name: Install python dependencies + shell: bash + working-directory: ${{ env.DIRECTORY }} + run: | + pip install -r requirements.txt + python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" + + - name: Integration test + shell: bash + working-directory: ${{ env.DIRECTORY }} + run: | + pytest . From 544b7653c2a51d2eb5228e97401524c15e3e56bf Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Thu, 1 Aug 2024 09:50:22 -0500 Subject: [PATCH 040/210] Make some deployments optional --- .github/workflows/cicd.yml | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 30635e1..91a8eda 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -19,6 +19,22 @@ concurrency: run-name: Deploy to ${{ inputs.environment }} environment by @${{ github.actor }} jobs: + export-env-vars: + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.environment }} + outputs: + DEPLOY_FEATURES_API: ${{ steps.init.outputs.DEPLOY_FEATURES_API }} + DEPLOY_AIRFLOW: ${{ steps.init.outputs.DEPLOY_AIRFLOW }} + DEPLOY_MONITORING: ${{ steps.init.outputs.DEPLOY_MONITORING }} + + steps: + - name: Environment variables to output + id: init + run: | + echo "DEPLOY_FEATURES_API=${{ vars.DEPLOY_FEATURES_API }}" >> $GITHUB_OUTPUT + echo "DEPLOY_AIRFLOW=${{ vars.DEPLOY_AIRFLOW }}" >> $GITHUB_OUTPUT + echo "DEPLOY_MONITORING=${{ vars.DEPLOY_MONITORING }}" >> $GITHUB_OUTPUT + deploy-veda-auth: name: Deploy VEDA auth πŸ” runs-on: ubuntu-latest @@ -121,7 +137,8 @@ jobs: DIRECTORY: veda-data-airflow AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [deploy-veda-auth, deploy-veda-backend] + needs: [deploy-veda-auth, deploy-veda-backend, export-env-vars] + if: needs.export-env-vars.DEPLOY_AIRFLOW environment: ${{ github.event.inputs.environment }} steps: @@ -158,11 +175,9 @@ jobs: deploy-veda-features-api: name: Deploy VEDA features-api πŸ—ΊοΈ runs-on: ubuntu-latest - if: ${{ vars.DEPLOY_FEATURES_API == true }} - env: - DIRECTORY: veda-features-api-cdk - ENVIRONMENT: ${{ github.event.inputs.environment }} environment: ${{ github.event.inputs.environment }} + needs: [export-env-vars] + if: needs.export-env-vars.DEPLOY_FEATURES_API steps: - name: Checkout @@ -206,13 +221,13 @@ jobs: deploy-veda-monitoring: name: deploy VEDA monitoring πŸ‘οΈ runs-on: ubuntu-latest - if: vars.DEPLOY_MONITORING env: DIRECTORY: veda-monitoring AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} - needs: [deploy-veda-auth, deploy-veda-backend, deploy-veda-data-airflow] + needs: [deploy-veda-backend, export-env-vars] + if: needs.export-env-vars.DEPLOY_AIRFLOW environment: ${{ github.event.inputs.environment }} steps: From 26e99d6e7f6255f33f5e6814cded29b5da793c06 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Thu, 1 Aug 2024 09:55:12 -0500 Subject: [PATCH 041/210] Fix outputs syntax --- .github/workflows/cicd.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 91a8eda..dcb0c53 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -138,7 +138,7 @@ jobs: AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} needs: [deploy-veda-auth, deploy-veda-backend, export-env-vars] - if: needs.export-env-vars.DEPLOY_AIRFLOW + if: needs.export-env-vars.outputs.DEPLOY_AIRFLOW environment: ${{ github.event.inputs.environment }} steps: @@ -177,7 +177,7 @@ jobs: runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment }} needs: [export-env-vars] - if: needs.export-env-vars.DEPLOY_FEATURES_API + if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API steps: - name: Checkout @@ -227,7 +227,7 @@ jobs: ENVIRONMENT: ${{ github.event.inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} needs: [deploy-veda-backend, export-env-vars] - if: needs.export-env-vars.DEPLOY_AIRFLOW + if: needs.export-env-vars.outputs.DEPLOY_AIRFLOW environment: ${{ github.event.inputs.environment }} steps: From f6e18fa6a4f4788b545987fcffe07764681f4a44 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Thu, 1 Aug 2024 09:58:27 -0500 Subject: [PATCH 042/210] Provide repo name --- .github/workflows/cicd.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index dcb0c53..51000a0 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -26,6 +26,7 @@ jobs: DEPLOY_FEATURES_API: ${{ steps.init.outputs.DEPLOY_FEATURES_API }} DEPLOY_AIRFLOW: ${{ steps.init.outputs.DEPLOY_AIRFLOW }} DEPLOY_MONITORING: ${{ steps.init.outputs.DEPLOY_MONITORING }} + DEPLOY_AUTH: ${{ steps.init.outputs.DEPLOY_AUTH }} steps: - name: Environment variables to output @@ -34,6 +35,7 @@ jobs: echo "DEPLOY_FEATURES_API=${{ vars.DEPLOY_FEATURES_API }}" >> $GITHUB_OUTPUT echo "DEPLOY_AIRFLOW=${{ vars.DEPLOY_AIRFLOW }}" >> $GITHUB_OUTPUT echo "DEPLOY_MONITORING=${{ vars.DEPLOY_MONITORING }}" >> $GITHUB_OUTPUT + echo "DEPLOY_AUTH=${{ vars.DEPLOY_AUTH }}" >> $GITHUB_OUTPUT deploy-veda-auth: name: Deploy VEDA auth πŸ” @@ -42,6 +44,8 @@ jobs: DIRECTORY: veda-auth ENVIRONMENT: ${{ github.event.inputs.environment }} environment: ${{ github.event.inputs.environment }} + needs: [export-env-vars] + if: needs.export-env-vars.outputs.DEPLOY_AUTH steps: - name: Checkout @@ -178,6 +182,9 @@ jobs: environment: ${{ github.event.inputs.environment }} needs: [export-env-vars] if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API + env: + DIRECTORY: veda-auth + ENVIRONMENT: ${{ github.event.inputs.environment }} steps: - name: Checkout From 4d52e87319f0a64143b6fb10df811e4f73cef106 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Thu, 1 Aug 2024 10:03:31 -0500 Subject: [PATCH 043/210] Fix repo name --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 51000a0..92b4f3f 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -183,7 +183,7 @@ jobs: needs: [export-env-vars] if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API env: - DIRECTORY: veda-auth + DIRECTORY: veda-features-api-cdk ENVIRONMENT: ${{ github.event.inputs.environment }} steps: From 83d07b729e69305e42c4f0e2af29ff607efea2c6 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Thu, 1 Aug 2024 10:08:38 -0500 Subject: [PATCH 044/210] Add submodule --- .gitmodules | 3 +++ veda-features-api-cdk | 1 + 2 files changed, 4 insertions(+) create mode 160000 veda-features-api-cdk diff --git a/.gitmodules b/.gitmodules index c2c9858..7e3e264 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "veda-monitoring"] path = veda-monitoring url = https://github.com/NASA-IMPACT/veda-monitoring.git +[submodule "veda-features-api-cdk"] + path = veda-features-api-cdk + url = git@github.com:NASA-IMPACT/veda-features-api-cdk.git diff --git a/veda-features-api-cdk b/veda-features-api-cdk new file mode 160000 index 0000000..8e32f0f --- /dev/null +++ b/veda-features-api-cdk @@ -0,0 +1 @@ +Subproject commit 8e32f0faecddb8d4d00e6b9b767132f5e8d286d3 From 89ce6a0848c814a6ad4832e8a9ad50121c795fa2 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Thu, 1 Aug 2024 10:24:59 -0500 Subject: [PATCH 045/210] Test removing submodule --- .gitmodules | 3 --- veda-features-api-cdk | 1 - 2 files changed, 4 deletions(-) delete mode 160000 veda-features-api-cdk diff --git a/.gitmodules b/.gitmodules index 7e3e264..c2c9858 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,6 +10,3 @@ [submodule "veda-monitoring"] path = veda-monitoring url = https://github.com/NASA-IMPACT/veda-monitoring.git -[submodule "veda-features-api-cdk"] - path = veda-features-api-cdk - url = git@github.com:NASA-IMPACT/veda-features-api-cdk.git diff --git a/veda-features-api-cdk b/veda-features-api-cdk deleted file mode 160000 index 8e32f0f..0000000 --- a/veda-features-api-cdk +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8e32f0faecddb8d4d00e6b9b767132f5e8d286d3 From cb47d13820b3e12ed2d11ab8c08af57205855fab Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Thu, 1 Aug 2024 10:27:00 -0500 Subject: [PATCH 046/210] Revert "Test removing submodule" This reverts commit 89ce6a0848c814a6ad4832e8a9ad50121c795fa2. --- .gitmodules | 3 +++ veda-features-api-cdk | 1 + 2 files changed, 4 insertions(+) create mode 160000 veda-features-api-cdk diff --git a/.gitmodules b/.gitmodules index c2c9858..7e3e264 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "veda-monitoring"] path = veda-monitoring url = https://github.com/NASA-IMPACT/veda-monitoring.git +[submodule "veda-features-api-cdk"] + path = veda-features-api-cdk + url = git@github.com:NASA-IMPACT/veda-features-api-cdk.git diff --git a/veda-features-api-cdk b/veda-features-api-cdk new file mode 160000 index 0000000..8e32f0f --- /dev/null +++ b/veda-features-api-cdk @@ -0,0 +1 @@ +Subproject commit 8e32f0faecddb8d4d00e6b9b767132f5e8d286d3 From 48075690406531306c99aeec8e96fafe428cdad5 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Thu, 1 Aug 2024 10:33:55 -0500 Subject: [PATCH 047/210] Change concurrency to environment --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 92b4f3f..60a6219 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -13,7 +13,7 @@ on: description: Environment to deploy to concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.environment }} + group: ${{ inputs.environment }} cancel-in-progress: true run-name: Deploy to ${{ inputs.environment }} environment by @${{ github.actor }} From a055c2aeff34a405e33974e0596a78970f37b222 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Thu, 1 Aug 2024 10:48:41 -0500 Subject: [PATCH 048/210] Add optional feature flag for everything --- .github/workflows/cicd.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 60a6219..dfb11fa 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -24,18 +24,20 @@ jobs: environment: ${{ github.event.inputs.environment }} outputs: DEPLOY_FEATURES_API: ${{ steps.init.outputs.DEPLOY_FEATURES_API }} - DEPLOY_AIRFLOW: ${{ steps.init.outputs.DEPLOY_AIRFLOW }} + DEPLOY_DATA_AIRFLOW: ${{ steps.init.outputs.DEPLOY_DATA_AIRFLOW }} DEPLOY_MONITORING: ${{ steps.init.outputs.DEPLOY_MONITORING }} DEPLOY_AUTH: ${{ steps.init.outputs.DEPLOY_AUTH }} + DEPLOY_BACKEND: ${{ steps.init.outputs.DEPLOY_BACKEND }} steps: - name: Environment variables to output id: init run: | echo "DEPLOY_FEATURES_API=${{ vars.DEPLOY_FEATURES_API }}" >> $GITHUB_OUTPUT - echo "DEPLOY_AIRFLOW=${{ vars.DEPLOY_AIRFLOW }}" >> $GITHUB_OUTPUT + echo "DEPLOY_DATA_AIRFLOW=${{ vars.DEPLOY_DATA_AIRFLOW }}" >> $GITHUB_OUTPUT echo "DEPLOY_MONITORING=${{ vars.DEPLOY_MONITORING }}" >> $GITHUB_OUTPUT echo "DEPLOY_AUTH=${{ vars.DEPLOY_AUTH }}" >> $GITHUB_OUTPUT + echo "DEPLOY_BACKEND=${{ vars.DEPLOY_BACKEND }}" >> $GITHUB_OUTPUT deploy-veda-auth: name: Deploy VEDA auth πŸ” @@ -91,7 +93,8 @@ jobs: env: DIRECTORY: veda-backend ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [deploy-veda-auth] + needs: [deploy-veda-auth, export-env-vars] + if: needs.export-env-vars.outputs.DEPLOY_BACKEND environment: ${{ github.event.inputs.environment }} steps: @@ -142,7 +145,7 @@ jobs: AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} needs: [deploy-veda-auth, deploy-veda-backend, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_AIRFLOW + if: needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW environment: ${{ github.event.inputs.environment }} steps: @@ -234,7 +237,7 @@ jobs: ENVIRONMENT: ${{ github.event.inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} needs: [deploy-veda-backend, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_AIRFLOW + if: needs.export-env-vars.outputs.DEPLOY_MONITORING environment: ${{ github.event.inputs.environment }} steps: From 1d163d8a65297e9280627f1ac3698dcefac86cb0 Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Thu, 1 Aug 2024 09:42:17 -0700 Subject: [PATCH 049/210] fix: remove cat command --- .github/workflows/cicd.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 8122d9e..b746a68 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -157,7 +157,6 @@ jobs: run: | workflows_api_value=$(cat ${HOME}/terraform_outputs.json) echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT - cat $GITHUB_OUTPUT deploy-veda-monitoring: name: deploy VEDA monitoring πŸ‘οΈ From 589cf64e4ece7490cf3cbde84e8e97cae7c4d779 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Thu, 1 Aug 2024 11:44:23 -0500 Subject: [PATCH 050/210] Make veda-auth and veda-backend deployment default to true --- .github/workflows/cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index dfb11fa..06b0f0a 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -47,7 +47,7 @@ jobs: ENVIRONMENT: ${{ github.event.inputs.environment }} environment: ${{ github.event.inputs.environment }} needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_AUTH + if: needs.export-env-vars.outputs.DEPLOY_AUTH != false steps: - name: Checkout @@ -94,7 +94,7 @@ jobs: DIRECTORY: veda-backend ENVIRONMENT: ${{ github.event.inputs.environment }} needs: [deploy-veda-auth, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_BACKEND + if: needs.export-env-vars.outputs.DEPLOY_BACKEND != false environment: ${{ github.event.inputs.environment }} steps: From 7f91bcc7a1255738a66af0b38b775ecfb6a09650 Mon Sep 17 00:00:00 2001 From: Slesa Adhikari Date: Thu, 1 Aug 2024 11:45:41 -0500 Subject: [PATCH 051/210] Change var type to string --- .github/workflows/cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 06b0f0a..ea268f3 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -47,7 +47,7 @@ jobs: ENVIRONMENT: ${{ github.event.inputs.environment }} environment: ${{ github.event.inputs.environment }} needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_AUTH != false + if: needs.export-env-vars.outputs.DEPLOY_AUTH != 'false' steps: - name: Checkout @@ -94,7 +94,7 @@ jobs: DIRECTORY: veda-backend ENVIRONMENT: ${{ github.event.inputs.environment }} needs: [deploy-veda-auth, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_BACKEND != false + if: needs.export-env-vars.outputs.DEPLOY_BACKEND != 'false' environment: ${{ github.event.inputs.environment }} steps: From 525b1d07954fde735976406620637efff68dd8a0 Mon Sep 17 00:00:00 2001 From: Jennifer Tran Date: Thu, 1 Aug 2024 09:48:48 -0700 Subject: [PATCH 052/210] feat: add outputs for deploy-veda-data-airflow job --- .github/workflows/cicd.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index b746a68..5b1f00d 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -158,6 +158,9 @@ jobs: workflows_api_value=$(cat ${HOME}/terraform_outputs.json) echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT + outputs: + workflows_api_endpoint: ${{ steps.get_workflows_api_endpoint.outputs.workflows_api_endpoint }} + deploy-veda-monitoring: name: deploy VEDA monitoring πŸ‘οΈ runs-on: ubuntu-latest From cf00898cde2674d41efb6c71ee3cccdefa0a951a Mon Sep 17 00:00:00 2001 From: Stephen Kilbourn Date: Fri, 2 Aug 2024 09:04:42 -0600 Subject: [PATCH 053/210] attempt to add playwright step (#27) * attempt to add playwright test step * checkout generate env script from deploy repo --- .github/workflows/cicd.yml | 63 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index ea268f3..011a99a 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -312,3 +312,66 @@ jobs: working-directory: ${{ env.DIRECTORY }} run: | pytest . + + run-playwright-checks: + name: playwright end to end tests + if: ${{ needs.define-environment.outputs.env_name }} + runs-on: ubuntu-latest + needs: [ deploy-veda-backend, deploy-veda-data-airflow] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + repository: NASA-IMPACT/veda-config + ref: add-playwright + + - name: Use Node.js 16 + uses: actions/setup-node@v3 + with: + node-version: 16 + + - name: Run veda setup + run: ./.veda/setup + + - name: Checkout generate_env script + uses: actions/checkout@v4 + with: + sparse-checkout: | + /scripts/generate_env_file.py + sparse-checkout-cone-mode: false + + - name: Generate .env file from cdk.out + shell: bash + working-directory: ${{ env.DIRECTORY }} + run: | + pip install -r requirements.txt + python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" + + - name: Load .env file + id: dotenv + uses: falti/dotenv-action@v1.1 + + - name: Create env.local file + env: + MAPBOX_TOKEN: ${{secrets.MAPBOX_TOKEN}} + run: | + touch .env.local + echo "API_RASTER_ENDPOINT=${{steps.dotenv.outputs.VEDA_RASTER_URL}}" >> .env.local + echo "API_STAC_ENDPOINT=${{steps.dotenv.outputs.VEDA_STAC_URL}}" >> .env.local + + echo "Testing deployment URLs" + cat .env.local + + echo "MAPBOX_TOKEN="$MAPBOX_TOKEN" >> .env.local + + - name: Install Playwright Browsers + run: yarn playwright install --with-deps + + - name: Playwright tests + run: yarn test:e2e + - uses: actions/upload-artifact@v3 + if: always() + with: + name: playwright-report + path: playwright-report/ + retention-days: 30 From 9c544649d8e412b18b05974933676aca0e376f3d Mon Sep 17 00:00:00 2001 From: Anish Bhusal Date: Tue, 6 Aug 2024 14:37:45 -0500 Subject: [PATCH 054/210] Add support to skip deployments for features-api and data-airflow (#43) * upgraded veda-backend to use v7.0.0 * check deploy flag for features-api * skip features api deployment by default * skip features api deployment by default * skip features api deployment by default * skip veda data airflow deployment by default unless true * Revert "upgraded veda-backend to use v7.0.0" This reverts commit 59033a7d6b570a070454930cafbb1aad9aa3f251. --- .github/workflows/cicd.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index aede1ed..cd6597d 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -47,7 +47,7 @@ jobs: ENVIRONMENT: ${{ github.event.inputs.environment }} environment: ${{ github.event.inputs.environment }} needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_AUTH != 'false' + if: needs.export-env-vars.outputs.DEPLOY_AUTH != 'false' steps: - name: Checkout @@ -94,7 +94,7 @@ jobs: DIRECTORY: veda-backend ENVIRONMENT: ${{ github.event.inputs.environment }} needs: [deploy-veda-auth, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_BACKEND != 'false' + if: needs.export-env-vars.outputs.DEPLOY_BACKEND != 'false' environment: ${{ github.event.inputs.environment }} steps: @@ -145,7 +145,7 @@ jobs: AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} needs: [deploy-veda-auth, deploy-veda-backend, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW + if: needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW && needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW != 'false' environment: ${{ github.event.inputs.environment }} steps: @@ -194,7 +194,7 @@ jobs: runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment }} needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API + if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API && needs.export-env-vars.outputs.DEPLOY_FEATURES_API != 'false' env: DIRECTORY: veda-features-api-cdk ENVIRONMENT: ${{ github.event.inputs.environment }} @@ -247,7 +247,7 @@ jobs: ENVIRONMENT: ${{ github.event.inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} needs: [deploy-veda-backend, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_MONITORING + if: needs.export-env-vars.outputs.DEPLOY_MONITORING != 'false' environment: ${{ github.event.inputs.environment }} steps: From 4dfa557f34e376dd2659067fd231c215a27d4094 Mon Sep 17 00:00:00 2001 From: Ciaran Sweet <9111975+ciaransweet@users.noreply.github.com> Date: Wed, 7 Aug 2024 17:14:18 +0100 Subject: [PATCH 055/210] feat: fetch outputs from veda-backend (#42) * feat: fetch outputs from veda-backend * Address review comment Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> * Undo rename of cdk outputs --------- Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- .github/workflows/cicd.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index cd6597d..6603d44 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -127,13 +127,25 @@ jobs: dir: "${{ env.DIRECTORY }}" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - - name: Get Backend Stack Name + - name: Get Backend Output Values id: get_backend_stack shell: bash run: | stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) echo "backend_stackname=$stack" >> $GITHUB_OUTPUT + raster_api_url=$(jq '.[keys_unsorted[0]].rasterapiurl' ${HOME}/cdk-outputs.json) + echo "raster_api_url=$raster_api_url" >> $GITHUB_OUTPUT + + ingest_api_url=$(jq '.[keys_unsorted[0]].ingestapiurl' ${HOME}/cdk-outputs.json) + echo "ingest_api_url=$ingest_api_url" >> $GITHUB_OUTPUT + + stac_api_url=$(jq '.[keys_unsorted[0]].stacapiurl' ${HOME}/cdk-outputs.json) + echo "stac_api_url=$stac_api_url" >> $GITHUB_OUTPUT + + stack_browser_url=$(jq '.[keys_unsorted[0]].stacbrowserurl' ${HOME}/cdk-outputs.json) + echo "stack_browser_url=$stack_browser_url" >> $GITHUB_OUTPUT + outputs: backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} From 0b9c9dbc6308b008c433f1aee92c66e905fc0fcc Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Wed, 7 Aug 2024 16:10:57 -0500 Subject: [PATCH 056/210] Add SM2A to GHG DEV --- README.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 501df91..2af3acd 100644 --- a/README.md +++ b/README.md @@ -43,4 +43,28 @@ STATE_BUCKET_KEY=***** STATE_DYNAMO_TABLE=***** VEDA_STAC_PATH_PREFIX=***** VEDA_RASTER_PATH_PREFIX=***** -``` \ No newline at end of file +``` + + +#### AWS Secrets Requirements For SM2A +```bash +AIRFLOW_UID=****** +PREFIX=****** +VPC_ID=****** +STATE_BUCKET_NAME=****** +STATE_BUCKET_KEY=****** +STATE_DYNAMO_TABLE=****** +PRIVATE_SUBNETS_TAGNAME=****** +PUBLIC_SUBNETS_TAGNAME=****** +AIRFLOW_FERNET_KEY=****** +AIRFLOW_DB_NAME=****** +AIRFLOW_DB_USERNAME=****** +AIRFLOW_DB_PASSWORD=****** +PERMISSION_BOUNDARIES_ARN=****** +DOMAIN_NAME=****** +STAGE=****** +TF_VAR_gh_app_client_id=****** +TF_VAR_gh_app_client_secret=****** +TF_VAR_gh_team_name=****** +TF_VAR_subdomain=****** +``` From 2589bb7497e4381e4a476de117d51fac280f31c1 Mon Sep 17 00:00:00 2001 From: Ciaran Sweet <9111975+ciaransweet@users.noreply.github.com> Date: Thu, 8 Aug 2024 16:49:24 +0100 Subject: [PATCH 057/210] fix: expose stac_browser_bucket_name (#44) --- .github/workflows/cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 6603d44..87fbcac 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -143,8 +143,8 @@ jobs: stac_api_url=$(jq '.[keys_unsorted[0]].stacapiurl' ${HOME}/cdk-outputs.json) echo "stac_api_url=$stac_api_url" >> $GITHUB_OUTPUT - stack_browser_url=$(jq '.[keys_unsorted[0]].stacbrowserurl' ${HOME}/cdk-outputs.json) - echo "stack_browser_url=$stack_browser_url" >> $GITHUB_OUTPUT + stack_browser_bucket_name=$(jq '.[keys_unsorted[0]].stacbrowserbucketname' ${HOME}/cdk-outputs.json) + echo "stack_browser_bucket_name=stack_browser_bucket_name" >> $GITHUB_OUTPUT outputs: backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} From 6d51674e3aa4ab7bde6a01f272042a152c80cd8d Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 9 Aug 2024 09:50:58 -0500 Subject: [PATCH 058/210] Only deploy SM2A --- .github/workflows/cicd.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 6603d44..e9d7b7e 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -180,17 +180,23 @@ jobs: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-airflow-deployment" aws-region: "${{ env.AWS_REGION }}" - - - name: Run deployment - uses: "./veda-data-airflow/.github/actions/terraform-deploy" +# - name: Run deployment +# uses: "./veda-data-airflow/.github/actions/terraform-deploy" +# with: +# env-file: ".env" +# env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} +# dir: "${{ env.DIRECTORY }}" +# script_path: "${{ github.workspace }}/scripts/generate_env_file.py" +# backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" +# auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + + - name: Run SM2A deployment + uses: "./veda-data-airflow/.github/actions/terraform-deploy-sm2a" with: env-file: ".env" - env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - dir: "${{ env.DIRECTORY }}" + env_aws_secret_name: ${{ vars.SM2A_ENVS_DEPLOYMENT_SECRET_NAME }} + dir: "${{ env.DIRECTORY }}/sm2a" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" - auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" - - name: Get Workflows API Endpoint id: get_workflows_api_endpoint shell: bash From 827febe0d84fcdf6220cc2452657430fc351fdaa Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 9 Aug 2024 13:59:42 -0500 Subject: [PATCH 059/210] Only deploy SM2A --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index e9d7b7e..7b1c632 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -156,7 +156,7 @@ jobs: DIRECTORY: veda-data-airflow AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [deploy-veda-auth, deploy-veda-backend, export-env-vars] + #needs: [deploy-veda-auth, deploy-veda-backend, export-env-vars] if: needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW && needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW != 'false' environment: ${{ github.event.inputs.environment }} From 8142fa0ab275823cde6476d89ef795140b8a1910 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 9 Aug 2024 14:07:34 -0500 Subject: [PATCH 060/210] Only deploy SM2A --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 7b1c632..211950a 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -156,7 +156,7 @@ jobs: DIRECTORY: veda-data-airflow AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} - #needs: [deploy-veda-auth, deploy-veda-backend, export-env-vars] + needs: [export-env-vars] #[deploy-veda-auth, deploy-veda-backend, export-env-vars] if: needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW && needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW != 'false' environment: ${{ github.event.inputs.environment }} From 604f6a2a5ca29cd4f1f49503a9468097c920dc15 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 9 Aug 2024 14:22:38 -0500 Subject: [PATCH 061/210] Add optional envfile --- scripts/generate_env_file.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/generate_env_file.py b/scripts/generate_env_file.py index 75536e4..7639c9e 100644 --- a/scripts/generate_env_file.py +++ b/scripts/generate_env_file.py @@ -54,15 +54,26 @@ def generate_env_file(secret_id, stack_names=None, out_file=".env"): help="Cloudformation Stack names (comma separated)", default=None, ) + parser.add_argument( + "--env-file", + dest="env_file", + help=".env file to write to", + required=False, + default=".env", + ) args = parser.parse_args() - secret_id, stack_names = ( + secret_id, stack_names, env_file = ( args.secret_id, - args.stack_names + args.stack_names, + args.env_file ) + + + generate_env_file( stack_names=stack_names, secret_id=secret_id, - out_file=".env" + out_file=env_file ) From e91ebbe5f41183bfadb0487b3dd28b493a0a7378 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 9 Aug 2024 15:27:22 -0500 Subject: [PATCH 062/210] Debug deployment --- .github/workflows/cicd.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 211950a..f1232e8 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -47,7 +47,7 @@ jobs: ENVIRONMENT: ${{ github.event.inputs.environment }} environment: ${{ github.event.inputs.environment }} needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_AUTH != 'false' + if: needs.export-env-vars.outputs.DEPLOY_AUTH = 'false_' steps: - name: Checkout @@ -94,7 +94,7 @@ jobs: DIRECTORY: veda-backend ENVIRONMENT: ${{ github.event.inputs.environment }} needs: [deploy-veda-auth, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_BACKEND != 'false' + if: needs.export-env-vars.outputs.DEPLOY_BACKEND = 'false_' environment: ${{ github.event.inputs.environment }} steps: @@ -157,7 +157,7 @@ jobs: AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} needs: [export-env-vars] #[deploy-veda-auth, deploy-veda-backend, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW && needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW != 'false' + #if: needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW && needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW = 'false_' environment: ${{ github.event.inputs.environment }} steps: @@ -172,7 +172,7 @@ jobs: with: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} - ref: ${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} + ref: "test-sm2a-deploy" # ${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -202,7 +202,8 @@ jobs: shell: bash run: | workflows_api_value=$(cat ${HOME}/terraform_outputs.json) - echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT + # echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT + echo $workflows_api_value outputs: workflows_api_endpoint: ${{ steps.get_workflows_api_endpoint.outputs.workflows_api_endpoint }} @@ -212,7 +213,7 @@ jobs: runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment }} needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API && needs.export-env-vars.outputs.DEPLOY_FEATURES_API != 'false' + if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API && needs.export-env-vars.outputs.DEPLOY_FEATURES_API = 'false_' env: DIRECTORY: veda-features-api-cdk ENVIRONMENT: ${{ github.event.inputs.environment }} @@ -265,7 +266,7 @@ jobs: ENVIRONMENT: ${{ github.event.inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} needs: [deploy-veda-backend, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_MONITORING != 'false' + if: needs.export-env-vars.outputs.DEPLOY_MONITORING = 'false_' environment: ${{ github.event.inputs.environment }} steps: From a57bb39e49f47d89487a72e12dc19a498c2f8649 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 9 Aug 2024 15:31:35 -0500 Subject: [PATCH 063/210] Debug deployment --- .github/workflows/cicd.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index f1232e8..6d2c6b4 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -47,7 +47,7 @@ jobs: ENVIRONMENT: ${{ github.event.inputs.environment }} environment: ${{ github.event.inputs.environment }} needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_AUTH = 'false_' + if: needs.export-env-vars.outputs.DEPLOY_AUTH = "false_" steps: - name: Checkout @@ -94,7 +94,7 @@ jobs: DIRECTORY: veda-backend ENVIRONMENT: ${{ github.event.inputs.environment }} needs: [deploy-veda-auth, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_BACKEND = 'false_' + if: needs.export-env-vars.outputs.DEPLOY_BACKEND = "false_" environment: ${{ github.event.inputs.environment }} steps: @@ -157,7 +157,7 @@ jobs: AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} needs: [export-env-vars] #[deploy-veda-auth, deploy-veda-backend, export-env-vars] - #if: needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW && needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW = 'false_' + #if: needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW && needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW = "false_" environment: ${{ github.event.inputs.environment }} steps: @@ -213,7 +213,7 @@ jobs: runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment }} needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API && needs.export-env-vars.outputs.DEPLOY_FEATURES_API = 'false_' + if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API && needs.export-env-vars.outputs.DEPLOY_FEATURES_API = "false_" env: DIRECTORY: veda-features-api-cdk ENVIRONMENT: ${{ github.event.inputs.environment }} @@ -266,7 +266,7 @@ jobs: ENVIRONMENT: ${{ github.event.inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} needs: [deploy-veda-backend, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_MONITORING = 'false_' + if: needs.export-env-vars.outputs.DEPLOY_MONITORING = "false_" environment: ${{ github.event.inputs.environment }} steps: From 21966ff9e76a4ff21c3dd9e3849cdf858b7b3779 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 9 Aug 2024 15:32:56 -0500 Subject: [PATCH 064/210] Debug deployment --- .github/workflows/cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 6d2c6b4..51beac2 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -47,7 +47,7 @@ jobs: ENVIRONMENT: ${{ github.event.inputs.environment }} environment: ${{ github.event.inputs.environment }} needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_AUTH = "false_" + if: needs.export-env-vars.outputs.DEPLOY_AUTH == "false_" steps: - name: Checkout @@ -94,7 +94,7 @@ jobs: DIRECTORY: veda-backend ENVIRONMENT: ${{ github.event.inputs.environment }} needs: [deploy-veda-auth, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_BACKEND = "false_" + if: needs.export-env-vars.outputs.DEPLOY_BACKEND == "false_" environment: ${{ github.event.inputs.environment }} steps: From 30d4d7c8ddaf2627923ac73bed4de8282c4b083c Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 9 Aug 2024 15:36:59 -0500 Subject: [PATCH 065/210] Only deploy SM2A job --- .github/workflows/cicd.yml | 612 ++++++++++++++++++------------------- 1 file changed, 306 insertions(+), 306 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 51beac2..16ca50b 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -39,115 +39,115 @@ jobs: echo "DEPLOY_AUTH=${{ vars.DEPLOY_AUTH }}" >> $GITHUB_OUTPUT echo "DEPLOY_BACKEND=${{ vars.DEPLOY_BACKEND }}" >> $GITHUB_OUTPUT - deploy-veda-auth: - name: Deploy VEDA auth πŸ” - runs-on: ubuntu-latest - env: - DIRECTORY: veda-auth - ENVIRONMENT: ${{ github.event.inputs.environment }} - environment: ${{ github.event.inputs.environment }} - needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_AUTH == "false_" - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - lfs: "true" - submodules: "false" - - - name: Checkout veda-auth submodule - uses: actions/checkout@v4 - with: - repository: "NASA-IMPACT/${{ env.DIRECTORY }}" - path: ${{ env.DIRECTORY }} - ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" - aws-region: "us-west-2" - - - name: Run auth deployment - id: deploy_auth_stack - uses: "./veda-auth/.github/actions/cdk-deploy" - with: - dir: "${{ env.DIRECTORY }}" - env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - - - name: Get Auth Stack Name - id: get_auth_stack - shell: bash - run: | - stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) - echo "auth_stackname=$stack" >> $GITHUB_OUTPUT - - outputs: - auth_stack_name: ${{ steps.get_auth_stack.outputs.auth_stackname }} - - deploy-veda-backend: - name: Deploy VEDA backend βš™οΈ - runs-on: ubuntu-latest - env: - DIRECTORY: veda-backend - ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [deploy-veda-auth, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_BACKEND == "false_" - environment: ${{ github.event.inputs.environment }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - lfs: "true" - submodules: "false" - - - name: Checkout veda-backend submodule - uses: actions/checkout@v4 - with: - repository: "NASA-IMPACT/${{ env.DIRECTORY }}" - path: ${{ env.DIRECTORY }} - ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main'}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "gh-${{ env.ENVIRONMENT }}-backend-deployment" - aws-region: "us-west-2" - - - name: Run deployment - uses: "./veda-backend/.github/actions/cdk-deploy" - id: deploy_backend_stack - with: - env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" - dir: "${{ env.DIRECTORY }}" - script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - - - name: Get Backend Output Values - id: get_backend_stack - shell: bash - run: | - stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) - echo "backend_stackname=$stack" >> $GITHUB_OUTPUT - - raster_api_url=$(jq '.[keys_unsorted[0]].rasterapiurl' ${HOME}/cdk-outputs.json) - echo "raster_api_url=$raster_api_url" >> $GITHUB_OUTPUT - - ingest_api_url=$(jq '.[keys_unsorted[0]].ingestapiurl' ${HOME}/cdk-outputs.json) - echo "ingest_api_url=$ingest_api_url" >> $GITHUB_OUTPUT - - stac_api_url=$(jq '.[keys_unsorted[0]].stacapiurl' ${HOME}/cdk-outputs.json) - echo "stac_api_url=$stac_api_url" >> $GITHUB_OUTPUT - - stack_browser_url=$(jq '.[keys_unsorted[0]].stacbrowserurl' ${HOME}/cdk-outputs.json) - echo "stack_browser_url=$stack_browser_url" >> $GITHUB_OUTPUT - - outputs: - backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} +# deploy-veda-auth: +# name: Deploy VEDA auth πŸ” +# runs-on: ubuntu-latest +# env: +# DIRECTORY: veda-auth +# ENVIRONMENT: ${{ github.event.inputs.environment }} +# environment: ${{ github.event.inputs.environment }} +# needs: [export-env-vars] +# if: ${{ needs.export-env-vars.outputs.DEPLOY_AUTH }} = "false_" +# +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# lfs: "true" +# submodules: "false" +# +# - name: Checkout veda-auth submodule +# uses: actions/checkout@v4 +# with: +# repository: "NASA-IMPACT/${{ env.DIRECTORY }}" +# path: ${{ env.DIRECTORY }} +# ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} +# +# - name: Configure AWS Credentials +# uses: aws-actions/configure-aws-credentials@v2 +# with: +# role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} +# role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" +# aws-region: "us-west-2" +# +# - name: Run auth deployment +# id: deploy_auth_stack +# uses: "./veda-auth/.github/actions/cdk-deploy" +# with: +# dir: "${{ env.DIRECTORY }}" +# env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} +# +# - name: Get Auth Stack Name +# id: get_auth_stack +# shell: bash +# run: | +# stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) +# echo "auth_stackname=$stack" >> $GITHUB_OUTPUT +# +# outputs: +# auth_stack_name: ${{ steps.get_auth_stack.outputs.auth_stackname }} + +# deploy-veda-backend: +# name: Deploy VEDA backend βš™οΈ +# runs-on: ubuntu-latest +# env: +# DIRECTORY: veda-backend +# ENVIRONMENT: ${{ github.event.inputs.environment }} +# needs: [deploy-veda-auth, export-env-vars] +# if: ${{ needs.export-env-vars.outputs.DEPLOY_BACKEND }} = "false_" +# environment: ${{ github.event.inputs.environment }} +# +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# lfs: "true" +# submodules: "false" +# +# - name: Checkout veda-backend submodule +# uses: actions/checkout@v4 +# with: +# repository: "NASA-IMPACT/${{ env.DIRECTORY }}" +# path: ${{ env.DIRECTORY }} +# ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main'}} +# +# - name: Configure AWS Credentials +# uses: aws-actions/configure-aws-credentials@v2 +# with: +# role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} +# role-session-name: "gh-${{ env.ENVIRONMENT }}-backend-deployment" +# aws-region: "us-west-2" +# +# - name: Run deployment +# uses: "./veda-backend/.github/actions/cdk-deploy" +# id: deploy_backend_stack +# with: +# env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} +# auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" +# dir: "${{ env.DIRECTORY }}" +# script_path: "${{ github.workspace }}/scripts/generate_env_file.py" +# +# - name: Get Backend Output Values +# id: get_backend_stack +# shell: bash +# run: | +# stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) +# echo "backend_stackname=$stack" >> $GITHUB_OUTPUT +# +# raster_api_url=$(jq '.[keys_unsorted[0]].rasterapiurl' ${HOME}/cdk-outputs.json) +# echo "raster_api_url=$raster_api_url" >> $GITHUB_OUTPUT +# +# ingest_api_url=$(jq '.[keys_unsorted[0]].ingestapiurl' ${HOME}/cdk-outputs.json) +# echo "ingest_api_url=$ingest_api_url" >> $GITHUB_OUTPUT +# +# stac_api_url=$(jq '.[keys_unsorted[0]].stacapiurl' ${HOME}/cdk-outputs.json) +# echo "stac_api_url=$stac_api_url" >> $GITHUB_OUTPUT +# +# stack_browser_url=$(jq '.[keys_unsorted[0]].stacbrowserurl' ${HOME}/cdk-outputs.json) +# echo "stack_browser_url=$stack_browser_url" >> $GITHUB_OUTPUT +# +# outputs: +# backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} deploy-veda-data-airflow: name: deploy VEDA data airflow πŸƒ @@ -207,200 +207,200 @@ jobs: outputs: workflows_api_endpoint: ${{ steps.get_workflows_api_endpoint.outputs.workflows_api_endpoint }} - - deploy-veda-features-api: - name: Deploy VEDA features-api πŸ—ΊοΈ - runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment }} - needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API && needs.export-env-vars.outputs.DEPLOY_FEATURES_API = "false_" - env: - DIRECTORY: veda-features-api-cdk - ENVIRONMENT: ${{ github.event.inputs.environment }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - lfs: "true" - submodules: "false" - - - name: Checkout veda-features-api submodule - uses: actions/checkout@v4 - with: - repository: "NASA-IMPACT/${{ env.DIRECTORY }}" - path: ${{ env.DIRECTORY }} - ref: ${{ vars.VEDA_FEATURES_API_GIT_REF || 'main'}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "gh-${{ env.ENVIRONMENT }}-features-deployment" - aws-region: "us-west-2" - - - name: Run deployment - uses: "./veda-features-api-cdk/.github/actions/cdk-deploy" - id: deploy_features_stack - with: - env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - dir: "${{ env.DIRECTORY }}" - script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - - - name: Get Features API Stack Name - id: get_features_api_stack - shell: bash - run: | - stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) - echo "features_api_stackname=$stack" >> $GITHUB_OUTPUT - - outputs: - features_api_stack_name: ${{ steps.get_features_api_stack.outputs.features_api_stackname }} - - deploy-veda-monitoring: - name: deploy VEDA monitoring πŸ‘οΈ - runs-on: ubuntu-latest - env: - DIRECTORY: veda-monitoring - AWS_REGION: "us-west-2" - ENVIRONMENT: ${{ github.event.inputs.environment }} - GH_PAT_CHECK: ${{ secrets.GH_PAT }} - needs: [deploy-veda-backend, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_MONITORING = "false_" - environment: ${{ github.event.inputs.environment }} - - steps: - - name: Checkout - if: ${{ env.GH_PAT_CHECK != '' }} - uses: actions/checkout@v4 - with: - lfs: "true" - submodules: "false" - - - name: Checkout veda-monitoring "submodule" - if: ${{ env.GH_PAT_CHECK != '' }} - uses: actions/checkout@v4 - with: - repository: "NASA-IMPACT/${{ env.DIRECTORY }}" - path: ${{ env.DIRECTORY }} - ref: ${{ vars.VEDA_MONITORING_GIT_REF || 'main' }} - token: ${{ secrets.GH_PAT }} - - - name: Configure AWS Credentials - if: ${{ env.GH_PAT_CHECK != '' }} - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "gh-${{ env.ENVIRONMENT }}-monitoring-deployment" - aws-region: "${{ env.AWS_REGION }}" - - - name: Run deployment - if: ${{ env.GH_PAT_CHECK != '' }} - uses: "./veda-monitoring/.github/actions/cdk-deploy" - with: - env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - dir: "${{ env.DIRECTORY }}" - script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - - test-deployment: - name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ - runs-on: ubuntu-latest - needs: [ deploy-veda-backend ] - env: - DIRECTORY: integration_test - ENVIRONMENT: ${{ github.event.inputs.environment }} - AWS_DEFAULT_REGION: us-west-2 - environment: ${{ github.event.inputs.environment }} - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - cache-dependency-path: | - ${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "gh-${{ env.ENVIRONMENT }}-integration-test" - aws-region: "${{ env.AWS_DEFAULT_REGION }}" - - - name: Install python dependencies - shell: bash - working-directory: ${{ env.DIRECTORY }} - run: | - pip install -r requirements.txt - python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" - - - name: Integration test - shell: bash - working-directory: ${{ env.DIRECTORY }} - run: | - pytest . - - run-playwright-checks: - name: playwright end to end tests - if: ${{ needs.define-environment.outputs.env_name }} - runs-on: ubuntu-latest - needs: [ deploy-veda-backend, deploy-veda-data-airflow] - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - repository: NASA-IMPACT/veda-config - ref: add-playwright - - - name: Use Node.js 16 - uses: actions/setup-node@v3 - with: - node-version: 16 - - - name: Run veda setup - run: ./.veda/setup - - - name: Checkout generate_env script - uses: actions/checkout@v4 - with: - sparse-checkout: | - /scripts/generate_env_file.py - sparse-checkout-cone-mode: false - - - name: Generate .env file from cdk.out - shell: bash - working-directory: ${{ env.DIRECTORY }} - run: | - pip install -r requirements.txt - python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" - - - name: Load .env file - id: dotenv - uses: falti/dotenv-action@v1.1 - - - name: Create env.local file - env: - MAPBOX_TOKEN: ${{secrets.MAPBOX_TOKEN}} - run: | - touch .env.local - echo "API_RASTER_ENDPOINT=${{steps.dotenv.outputs.VEDA_RASTER_URL}}" >> .env.local - echo "API_STAC_ENDPOINT=${{steps.dotenv.outputs.VEDA_STAC_URL}}" >> .env.local - - echo "Testing deployment URLs" - cat .env.local - - echo "MAPBOX_TOKEN="$MAPBOX_TOKEN" >> .env.local - - - name: Install Playwright Browsers - run: yarn playwright install --with-deps - - - name: Playwright tests - run: yarn test:e2e - - uses: actions/upload-artifact@v3 - if: always() - with: - name: playwright-report - path: playwright-report/ - retention-days: 30 +# +# deploy-veda-features-api: +# name: Deploy VEDA features-api πŸ—ΊοΈ +# runs-on: ubuntu-latest +# environment: ${{ github.event.inputs.environment }} +# needs: [export-env-vars] +# if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API && needs.export-env-vars.outputs.DEPLOY_FEATURES_API = "false_" +# env: +# DIRECTORY: veda-features-api-cdk +# ENVIRONMENT: ${{ github.event.inputs.environment }} +# +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# lfs: "true" +# submodules: "false" +# +# - name: Checkout veda-features-api submodule +# uses: actions/checkout@v4 +# with: +# repository: "NASA-IMPACT/${{ env.DIRECTORY }}" +# path: ${{ env.DIRECTORY }} +# ref: ${{ vars.VEDA_FEATURES_API_GIT_REF || 'main'}} +# +# - name: Configure AWS Credentials +# uses: aws-actions/configure-aws-credentials@v2 +# with: +# role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} +# role-session-name: "gh-${{ env.ENVIRONMENT }}-features-deployment" +# aws-region: "us-west-2" +# +# - name: Run deployment +# uses: "./veda-features-api-cdk/.github/actions/cdk-deploy" +# id: deploy_features_stack +# with: +# env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} +# dir: "${{ env.DIRECTORY }}" +# script_path: "${{ github.workspace }}/scripts/generate_env_file.py" +# +# - name: Get Features API Stack Name +# id: get_features_api_stack +# shell: bash +# run: | +# stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) +# echo "features_api_stackname=$stack" >> $GITHUB_OUTPUT +# +# outputs: +# features_api_stack_name: ${{ steps.get_features_api_stack.outputs.features_api_stackname }} +# +# deploy-veda-monitoring: +# name: deploy VEDA monitoring πŸ‘οΈ +# runs-on: ubuntu-latest +# env: +# DIRECTORY: veda-monitoring +# AWS_REGION: "us-west-2" +# ENVIRONMENT: ${{ github.event.inputs.environment }} +# GH_PAT_CHECK: ${{ secrets.GH_PAT }} +# needs: [deploy-veda-backend, export-env-vars] +# if: needs.export-env-vars.outputs.DEPLOY_MONITORING = "false_" +# environment: ${{ github.event.inputs.environment }} +# +# steps: +# - name: Checkout +# if: ${{ env.GH_PAT_CHECK != '' }} +# uses: actions/checkout@v4 +# with: +# lfs: "true" +# submodules: "false" +# +# - name: Checkout veda-monitoring "submodule" +# if: ${{ env.GH_PAT_CHECK != '' }} +# uses: actions/checkout@v4 +# with: +# repository: "NASA-IMPACT/${{ env.DIRECTORY }}" +# path: ${{ env.DIRECTORY }} +# ref: ${{ vars.VEDA_MONITORING_GIT_REF || 'main' }} +# token: ${{ secrets.GH_PAT }} +# +# - name: Configure AWS Credentials +# if: ${{ env.GH_PAT_CHECK != '' }} +# uses: aws-actions/configure-aws-credentials@v2 +# with: +# role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} +# role-session-name: "gh-${{ env.ENVIRONMENT }}-monitoring-deployment" +# aws-region: "${{ env.AWS_REGION }}" +# +# - name: Run deployment +# if: ${{ env.GH_PAT_CHECK != '' }} +# uses: "./veda-monitoring/.github/actions/cdk-deploy" +# with: +# env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} +# dir: "${{ env.DIRECTORY }}" +# script_path: "${{ github.workspace }}/scripts/generate_env_file.py" +# +# test-deployment: +# name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ +# runs-on: ubuntu-latest +# needs: [ deploy-veda-backend ] +# env: +# DIRECTORY: integration_test +# ENVIRONMENT: ${{ github.event.inputs.environment }} +# AWS_DEFAULT_REGION: us-west-2 +# environment: ${{ github.event.inputs.environment }} +# steps: +# - uses: actions/checkout@v2 +# +# - name: Set up Python +# uses: actions/setup-python@v4 +# with: +# python-version: "3.10" +# cache: "pip" +# cache-dependency-path: | +# ${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt +# +# - name: Configure AWS Credentials +# uses: aws-actions/configure-aws-credentials@v2 +# with: +# role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} +# role-session-name: "gh-${{ env.ENVIRONMENT }}-integration-test" +# aws-region: "${{ env.AWS_DEFAULT_REGION }}" +# +# - name: Install python dependencies +# shell: bash +# working-directory: ${{ env.DIRECTORY }} +# run: | +# pip install -r requirements.txt +# python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" +# +# - name: Integration test +# shell: bash +# working-directory: ${{ env.DIRECTORY }} +# run: | +# pytest . +# +# run-playwright-checks: +# name: playwright end to end tests +# if: ${{ needs.define-environment.outputs.env_name }} +# runs-on: ubuntu-latest +# needs: [ deploy-veda-backend, deploy-veda-data-airflow] +# steps: +# - name: Checkout +# uses: actions/checkout@v4 +# with: +# repository: NASA-IMPACT/veda-config +# ref: add-playwright +# +# - name: Use Node.js 16 +# uses: actions/setup-node@v3 +# with: +# node-version: 16 +# +# - name: Run veda setup +# run: ./.veda/setup +# +# - name: Checkout generate_env script +# uses: actions/checkout@v4 +# with: +# sparse-checkout: | +# /scripts/generate_env_file.py +# sparse-checkout-cone-mode: false +# +# - name: Generate .env file from cdk.out +# shell: bash +# working-directory: ${{ env.DIRECTORY }} +# run: | +# pip install -r requirements.txt +# python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" +# +# - name: Load .env file +# id: dotenv +# uses: falti/dotenv-action@v1.1 +# +# - name: Create env.local file +# env: +# MAPBOX_TOKEN: ${{secrets.MAPBOX_TOKEN}} +# run: | +# touch .env.local +# echo "API_RASTER_ENDPOINT=${{steps.dotenv.outputs.VEDA_RASTER_URL}}" >> .env.local +# echo "API_STAC_ENDPOINT=${{steps.dotenv.outputs.VEDA_STAC_URL}}" >> .env.local +# +# echo "Testing deployment URLs" +# cat .env.local +# +# echo "MAPBOX_TOKEN="$MAPBOX_TOKEN" >> .env.local +# +# - name: Install Playwright Browsers +# run: yarn playwright install --with-deps +# +# - name: Playwright tests +# run: yarn test:e2e +# - uses: actions/upload-artifact@v3 +# if: always() +# with: +# name: playwright-report +# path: playwright-report/ +# retention-days: 30 From 8ae41bf693d43bddf8c93ad4e22ab0b4077b6f74 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 9 Aug 2024 15:58:41 -0500 Subject: [PATCH 066/210] Add print to debug --- .github/workflows/cicd.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 16ca50b..bcd3df0 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -201,9 +201,11 @@ jobs: id: get_workflows_api_endpoint shell: bash run: | + ls -al ${HOME} workflows_api_value=$(cat ${HOME}/terraform_outputs.json) # echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT echo $workflows_api_value + echo "Done" outputs: workflows_api_endpoint: ${{ steps.get_workflows_api_endpoint.outputs.workflows_api_endpoint }} From 38cd7055670549b1beaeeb7cffed8c8454fa8196 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 9 Aug 2024 16:01:54 -0500 Subject: [PATCH 067/210] Add print to debug --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index bcd3df0..e01a240 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -204,7 +204,7 @@ jobs: ls -al ${HOME} workflows_api_value=$(cat ${HOME}/terraform_outputs.json) # echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT - echo $workflows_api_value + echo ${workflows_api_value} echo "Done" outputs: From 0b56b01d79767bc84ce7547a5085cc113ebdbd78 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 9 Aug 2024 16:06:36 -0500 Subject: [PATCH 068/210] Add print to debug --- .github/workflows/cicd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index e01a240..00d56ca 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -202,6 +202,7 @@ jobs: shell: bash run: | ls -al ${HOME} + cat ${HOME}/terraform_outputs.json workflows_api_value=$(cat ${HOME}/terraform_outputs.json) # echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT echo ${workflows_api_value} From e09a119907a7e5321d9405675108eae648cb4459 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 9 Aug 2024 16:18:35 -0500 Subject: [PATCH 069/210] Add print to debug --- .github/workflows/cicd.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 00d56ca..8d8892a 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -191,25 +191,16 @@ jobs: # auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" - name: Run SM2A deployment + id: deploy_sm2a uses: "./veda-data-airflow/.github/actions/terraform-deploy-sm2a" with: env-file: ".env" env_aws_secret_name: ${{ vars.SM2A_ENVS_DEPLOYMENT_SECRET_NAME }} dir: "${{ env.DIRECTORY }}/sm2a" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - - name: Get Workflows API Endpoint - id: get_workflows_api_endpoint - shell: bash - run: | - ls -al ${HOME} - cat ${HOME}/terraform_outputs.json - workflows_api_value=$(cat ${HOME}/terraform_outputs.json) - # echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT - echo ${workflows_api_value} - echo "Done" outputs: - workflows_api_endpoint: ${{ steps.get_workflows_api_endpoint.outputs.workflows_api_endpoint }} + workflows_api_endpoint: ${{ steps.deploy_sm2a.outputs.workflows_api_endpoint }} # # deploy-veda-features-api: # name: Deploy VEDA features-api πŸ—ΊοΈ From 185b24a811d21d62281ce1fe068f30d1ca89c0c9 Mon Sep 17 00:00:00 2001 From: Ciaran Sweet <9111975+ciaransweet@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:58:56 +0100 Subject: [PATCH 070/210] fix: add job outputs for veda-backend (#45) --- .github/workflows/cicd.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 87fbcac..34a292d 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -148,6 +148,10 @@ jobs: outputs: backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} + raster_api_url: ${{ steps.get_backend_stack.outputs.raster_api_url }} + ingest_api_url: ${{ steps.get_backend_stack.outputs.ingest_api_url }} + stac_api_url: ${{ steps.get_backend_stack.outputs.stac_api_url }} + stack_browser_bucket_name: ${{ steps.get_backend_stack.outputs.stack_browser_bucket_name }} deploy-veda-data-airflow: name: deploy VEDA data airflow πŸƒ From 29f4748c63178924edbaf4d55e33b40b1ee6eebd Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Tue, 13 Aug 2024 11:54:07 -0500 Subject: [PATCH 071/210] Fix github output --- .github/workflows/cicd.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 8d8892a..71e9eab 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -199,8 +199,15 @@ jobs: dir: "${{ env.DIRECTORY }}/sm2a" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + - name: Get Workflows API Endpoint + id: get_sm2a_workflows_api_endpoint + shell: bash + run: | + workflows_sm2a_api_value=$(cat ${HOME}/terraform_outputs.json) + echo "workflows_sm2a_api_value=$workflows_sm2a_api_value" >> $GITHUB_OUTPUT + outputs: - workflows_api_endpoint: ${{ steps.deploy_sm2a.outputs.workflows_api_endpoint }} + workflows_sm2a_api_value: ${{ steps.get_sm2a_workflows_api_endpoint.outputs.workflows_sm2a_api_value }} # # deploy-veda-features-api: # name: Deploy VEDA features-api πŸ—ΊοΈ From e6191a57b1cfb7f9ef32d1236376dd19754ec706 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Tue, 13 Aug 2024 11:56:36 -0500 Subject: [PATCH 072/210] Fix github tag --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 71e9eab..81acf0c 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -172,7 +172,7 @@ jobs: with: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} - ref: "test-sm2a-deploy" # ${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} + ref: "feature/add-sm2a-to-veda-pipeline" # ${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 From 03e6d04ff34e364f7345f7b4cc97084d29189e81 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Tue, 13 Aug 2024 12:25:53 -0500 Subject: [PATCH 073/210] Fix github output --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 81acf0c..15bcd30 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -203,7 +203,7 @@ jobs: id: get_sm2a_workflows_api_endpoint shell: bash run: | - workflows_sm2a_api_value=$(cat ${HOME}/terraform_outputs.json) + workflows_sm2a_api_value=$(cat ${HOME}/output_sm2a_workflows_endpoint.json | tr -d '"') echo "workflows_sm2a_api_value=$workflows_sm2a_api_value" >> $GITHUB_OUTPUT outputs: From e9c91c406dc3ceb2beec5733db0871596d506ec1 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Tue, 13 Aug 2024 12:30:54 -0500 Subject: [PATCH 074/210] Use set-output --- .github/workflows/cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 15bcd30..d82973f 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -203,8 +203,8 @@ jobs: id: get_sm2a_workflows_api_endpoint shell: bash run: | - workflows_sm2a_api_value=$(cat ${HOME}/output_sm2a_workflows_endpoint.json | tr -d '"') - echo "workflows_sm2a_api_value=$workflows_sm2a_api_value" >> $GITHUB_OUTPUT + workflows_sm2a_api_value=$(cat ${HOME}/output_sm2a_workflows_endpoint.json) + echo "::set-output name=workflows_sm2a_api_value::$workflows_sm2a_api_value" outputs: workflows_sm2a_api_value: ${{ steps.get_sm2a_workflows_api_endpoint.outputs.workflows_sm2a_api_value }} From 3e7966bb0e6a779f345d1d20b5a4bddabc5ca9fd Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Tue, 13 Aug 2024 12:38:10 -0500 Subject: [PATCH 075/210] Use recomanded way of setting the output --- .github/workflows/cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index d82973f..a5414ef 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -203,8 +203,8 @@ jobs: id: get_sm2a_workflows_api_endpoint shell: bash run: | - workflows_sm2a_api_value=$(cat ${HOME}/output_sm2a_workflows_endpoint.json) - echo "::set-output name=workflows_sm2a_api_value::$workflows_sm2a_api_value" + workflows_sm2a_api_value=$(cat ${HOME}/output_sm2a_workflows_endpoint.json | tr -d '"' |tr -d '\n' ) + echo "workflows_sm2a_api_value=$workflows_sm2a_api_value" >> $GITHUB_OUTPUT outputs: workflows_sm2a_api_value: ${{ steps.get_sm2a_workflows_api_endpoint.outputs.workflows_sm2a_api_value }} From a408550d12869ec9e79df1c3cac4579462cb4eb0 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Tue, 13 Aug 2024 13:17:42 -0500 Subject: [PATCH 076/210] Add SM2A to GHG DEV --- .github/workflows/cicd.yml | 656 +++++++++++++++++++------------------ 1 file changed, 335 insertions(+), 321 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index a5414ef..70e464c 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -39,115 +39,119 @@ jobs: echo "DEPLOY_AUTH=${{ vars.DEPLOY_AUTH }}" >> $GITHUB_OUTPUT echo "DEPLOY_BACKEND=${{ vars.DEPLOY_BACKEND }}" >> $GITHUB_OUTPUT -# deploy-veda-auth: -# name: Deploy VEDA auth πŸ” -# runs-on: ubuntu-latest -# env: -# DIRECTORY: veda-auth -# ENVIRONMENT: ${{ github.event.inputs.environment }} -# environment: ${{ github.event.inputs.environment }} -# needs: [export-env-vars] -# if: ${{ needs.export-env-vars.outputs.DEPLOY_AUTH }} = "false_" -# -# steps: -# - name: Checkout -# uses: actions/checkout@v4 -# with: -# lfs: "true" -# submodules: "false" -# -# - name: Checkout veda-auth submodule -# uses: actions/checkout@v4 -# with: -# repository: "NASA-IMPACT/${{ env.DIRECTORY }}" -# path: ${{ env.DIRECTORY }} -# ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} -# -# - name: Configure AWS Credentials -# uses: aws-actions/configure-aws-credentials@v2 -# with: -# role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} -# role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" -# aws-region: "us-west-2" -# -# - name: Run auth deployment -# id: deploy_auth_stack -# uses: "./veda-auth/.github/actions/cdk-deploy" -# with: -# dir: "${{ env.DIRECTORY }}" -# env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} -# -# - name: Get Auth Stack Name -# id: get_auth_stack -# shell: bash -# run: | -# stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) -# echo "auth_stackname=$stack" >> $GITHUB_OUTPUT -# -# outputs: -# auth_stack_name: ${{ steps.get_auth_stack.outputs.auth_stackname }} - -# deploy-veda-backend: -# name: Deploy VEDA backend βš™οΈ -# runs-on: ubuntu-latest -# env: -# DIRECTORY: veda-backend -# ENVIRONMENT: ${{ github.event.inputs.environment }} -# needs: [deploy-veda-auth, export-env-vars] -# if: ${{ needs.export-env-vars.outputs.DEPLOY_BACKEND }} = "false_" -# environment: ${{ github.event.inputs.environment }} -# -# steps: -# - name: Checkout -# uses: actions/checkout@v4 -# with: -# lfs: "true" -# submodules: "false" -# -# - name: Checkout veda-backend submodule -# uses: actions/checkout@v4 -# with: -# repository: "NASA-IMPACT/${{ env.DIRECTORY }}" -# path: ${{ env.DIRECTORY }} -# ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main'}} -# -# - name: Configure AWS Credentials -# uses: aws-actions/configure-aws-credentials@v2 -# with: -# role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} -# role-session-name: "gh-${{ env.ENVIRONMENT }}-backend-deployment" -# aws-region: "us-west-2" -# -# - name: Run deployment -# uses: "./veda-backend/.github/actions/cdk-deploy" -# id: deploy_backend_stack -# with: -# env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} -# auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" -# dir: "${{ env.DIRECTORY }}" -# script_path: "${{ github.workspace }}/scripts/generate_env_file.py" -# -# - name: Get Backend Output Values -# id: get_backend_stack -# shell: bash -# run: | -# stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) -# echo "backend_stackname=$stack" >> $GITHUB_OUTPUT -# -# raster_api_url=$(jq '.[keys_unsorted[0]].rasterapiurl' ${HOME}/cdk-outputs.json) -# echo "raster_api_url=$raster_api_url" >> $GITHUB_OUTPUT -# -# ingest_api_url=$(jq '.[keys_unsorted[0]].ingestapiurl' ${HOME}/cdk-outputs.json) -# echo "ingest_api_url=$ingest_api_url" >> $GITHUB_OUTPUT -# -# stac_api_url=$(jq '.[keys_unsorted[0]].stacapiurl' ${HOME}/cdk-outputs.json) -# echo "stac_api_url=$stac_api_url" >> $GITHUB_OUTPUT -# -# stack_browser_url=$(jq '.[keys_unsorted[0]].stacbrowserurl' ${HOME}/cdk-outputs.json) -# echo "stack_browser_url=$stack_browser_url" >> $GITHUB_OUTPUT -# -# outputs: -# backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} + deploy-veda-auth: + name: Deploy VEDA auth πŸ” + runs-on: ubuntu-latest + env: + DIRECTORY: veda-auth + ENVIRONMENT: ${{ github.event.inputs.environment }} + environment: ${{ github.event.inputs.environment }} + needs: [export-env-vars] + if: needs.export-env-vars.outputs.DEPLOY_AUTH != 'false' + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: "true" + submodules: "false" + + - name: Checkout veda-auth submodule + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" + aws-region: "us-west-2" + + - name: Run auth deployment + id: deploy_auth_stack + uses: "./veda-auth/.github/actions/cdk-deploy" + with: + dir: "${{ env.DIRECTORY }}" + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + + - name: Get Auth Stack Name + id: get_auth_stack + shell: bash + run: | + stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) + echo "auth_stackname=$stack" >> $GITHUB_OUTPUT + + outputs: + auth_stack_name: ${{ steps.get_auth_stack.outputs.auth_stackname }} + + deploy-veda-backend: + name: Deploy VEDA backend βš™οΈ + runs-on: ubuntu-latest + env: + DIRECTORY: veda-backend + ENVIRONMENT: ${{ github.event.inputs.environment }} + needs: [deploy-veda-auth, export-env-vars] + if: needs.export-env-vars.outputs.DEPLOY_BACKEND != 'false' + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: "true" + submodules: "false" + + - name: Checkout veda-backend submodule + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main'}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-backend-deployment" + aws-region: "us-west-2" + + - name: Run deployment + uses: "./veda-backend/.github/actions/cdk-deploy" + id: deploy_backend_stack + with: + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + dir: "${{ env.DIRECTORY }}" + script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + + - name: Get Backend Output Values + id: get_backend_stack + shell: bash + run: | + stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) + echo "backend_stackname=$stack" >> $GITHUB_OUTPUT + + raster_api_url=$(jq '.[keys_unsorted[0]].rasterapiurl' ${HOME}/cdk-outputs.json) + echo "raster_api_url=$raster_api_url" >> $GITHUB_OUTPUT + + ingest_api_url=$(jq '.[keys_unsorted[0]].ingestapiurl' ${HOME}/cdk-outputs.json) + echo "ingest_api_url=$ingest_api_url" >> $GITHUB_OUTPUT + + stac_api_url=$(jq '.[keys_unsorted[0]].stacapiurl' ${HOME}/cdk-outputs.json) + echo "stac_api_url=$stac_api_url" >> $GITHUB_OUTPUT + + stack_browser_bucket_name=$(jq '.[keys_unsorted[0]].stacbrowserbucketname' ${HOME}/cdk-outputs.json) + echo "stack_browser_bucket_name=stack_browser_bucket_name" >> $GITHUB_OUTPUT + + outputs: + backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} + raster_api_url: ${{ steps.get_backend_stack.outputs.raster_api_url }} + ingest_api_url: ${{ steps.get_backend_stack.outputs.ingest_api_url }} + stac_api_url: ${{ steps.get_backend_stack.outputs.stac_api_url }} + stack_browser_bucket_name: ${{ steps.get_backend_stack.outputs.stack_browser_bucket_name }} deploy-veda-data-airflow: name: deploy VEDA data airflow πŸƒ @@ -156,8 +160,8 @@ jobs: DIRECTORY: veda-data-airflow AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [export-env-vars] #[deploy-veda-auth, deploy-veda-backend, export-env-vars] - #if: needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW && needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW = "false_" + needs: [deploy-veda-auth, deploy-veda-backend, export-env-vars] + if: needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW && needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW != 'false' environment: ${{ github.event.inputs.environment }} steps: @@ -172,7 +176,7 @@ jobs: with: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} - ref: "feature/add-sm2a-to-veda-pipeline" # ${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} + ref: ${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -180,18 +184,27 @@ jobs: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-airflow-deployment" aws-region: "${{ env.AWS_REGION }}" -# - name: Run deployment -# uses: "./veda-data-airflow/.github/actions/terraform-deploy" -# with: -# env-file: ".env" -# env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} -# dir: "${{ env.DIRECTORY }}" -# script_path: "${{ github.workspace }}/scripts/generate_env_file.py" -# backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" -# auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + + - name: Run MWAA deployment + uses: "./veda-data-airflow/.github/actions/terraform-deploy" + with: + env-file: ".env" + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + dir: "${{ env.DIRECTORY }}" + script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" + auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + + - name: Get Workflows API Endpoint + id: get_workflows_api_endpoint + shell: bash + run: | + workflows_api_value=$(cat ${HOME}/terraform_outputs.json) + echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT - name: Run SM2A deployment id: deploy_sm2a + if: ${{ vars.DEPLOY_SM2A }} = "true" uses: "./veda-data-airflow/.github/actions/terraform-deploy-sm2a" with: env-file: ".env" @@ -199,209 +212,210 @@ jobs: dir: "${{ env.DIRECTORY }}/sm2a" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - - name: Get Workflows API Endpoint + - name: Get Workflows SM2A API Endpoint id: get_sm2a_workflows_api_endpoint shell: bash run: | - workflows_sm2a_api_value=$(cat ${HOME}/output_sm2a_workflows_endpoint.json | tr -d '"' |tr -d '\n' ) - echo "workflows_sm2a_api_value=$workflows_sm2a_api_value" >> $GITHUB_OUTPUT + workflows_sm2a_api_value=$(cat ${HOME}/output_sm2a_workflows_endpoint.json 2>/dev/null | tr -d '"' | tr -d '\n' ) + echo "workflows_sm2a_api_value=${workflows_sm2a_api_value:-NA}" >> $GITHUB_OUTPUT outputs: + workflows_api_endpoint: ${{ steps.get_workflows_api_endpoint.outputs.workflows_api_endpoint }} workflows_sm2a_api_value: ${{ steps.get_sm2a_workflows_api_endpoint.outputs.workflows_sm2a_api_value }} -# -# deploy-veda-features-api: -# name: Deploy VEDA features-api πŸ—ΊοΈ -# runs-on: ubuntu-latest -# environment: ${{ github.event.inputs.environment }} -# needs: [export-env-vars] -# if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API && needs.export-env-vars.outputs.DEPLOY_FEATURES_API = "false_" -# env: -# DIRECTORY: veda-features-api-cdk -# ENVIRONMENT: ${{ github.event.inputs.environment }} -# -# steps: -# - name: Checkout -# uses: actions/checkout@v4 -# with: -# lfs: "true" -# submodules: "false" -# -# - name: Checkout veda-features-api submodule -# uses: actions/checkout@v4 -# with: -# repository: "NASA-IMPACT/${{ env.DIRECTORY }}" -# path: ${{ env.DIRECTORY }} -# ref: ${{ vars.VEDA_FEATURES_API_GIT_REF || 'main'}} -# -# - name: Configure AWS Credentials -# uses: aws-actions/configure-aws-credentials@v2 -# with: -# role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} -# role-session-name: "gh-${{ env.ENVIRONMENT }}-features-deployment" -# aws-region: "us-west-2" -# -# - name: Run deployment -# uses: "./veda-features-api-cdk/.github/actions/cdk-deploy" -# id: deploy_features_stack -# with: -# env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} -# dir: "${{ env.DIRECTORY }}" -# script_path: "${{ github.workspace }}/scripts/generate_env_file.py" -# -# - name: Get Features API Stack Name -# id: get_features_api_stack -# shell: bash -# run: | -# stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) -# echo "features_api_stackname=$stack" >> $GITHUB_OUTPUT -# -# outputs: -# features_api_stack_name: ${{ steps.get_features_api_stack.outputs.features_api_stackname }} -# -# deploy-veda-monitoring: -# name: deploy VEDA monitoring πŸ‘οΈ -# runs-on: ubuntu-latest -# env: -# DIRECTORY: veda-monitoring -# AWS_REGION: "us-west-2" -# ENVIRONMENT: ${{ github.event.inputs.environment }} -# GH_PAT_CHECK: ${{ secrets.GH_PAT }} -# needs: [deploy-veda-backend, export-env-vars] -# if: needs.export-env-vars.outputs.DEPLOY_MONITORING = "false_" -# environment: ${{ github.event.inputs.environment }} -# -# steps: -# - name: Checkout -# if: ${{ env.GH_PAT_CHECK != '' }} -# uses: actions/checkout@v4 -# with: -# lfs: "true" -# submodules: "false" -# -# - name: Checkout veda-monitoring "submodule" -# if: ${{ env.GH_PAT_CHECK != '' }} -# uses: actions/checkout@v4 -# with: -# repository: "NASA-IMPACT/${{ env.DIRECTORY }}" -# path: ${{ env.DIRECTORY }} -# ref: ${{ vars.VEDA_MONITORING_GIT_REF || 'main' }} -# token: ${{ secrets.GH_PAT }} -# -# - name: Configure AWS Credentials -# if: ${{ env.GH_PAT_CHECK != '' }} -# uses: aws-actions/configure-aws-credentials@v2 -# with: -# role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} -# role-session-name: "gh-${{ env.ENVIRONMENT }}-monitoring-deployment" -# aws-region: "${{ env.AWS_REGION }}" -# -# - name: Run deployment -# if: ${{ env.GH_PAT_CHECK != '' }} -# uses: "./veda-monitoring/.github/actions/cdk-deploy" -# with: -# env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} -# dir: "${{ env.DIRECTORY }}" -# script_path: "${{ github.workspace }}/scripts/generate_env_file.py" -# -# test-deployment: -# name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ -# runs-on: ubuntu-latest -# needs: [ deploy-veda-backend ] -# env: -# DIRECTORY: integration_test -# ENVIRONMENT: ${{ github.event.inputs.environment }} -# AWS_DEFAULT_REGION: us-west-2 -# environment: ${{ github.event.inputs.environment }} -# steps: -# - uses: actions/checkout@v2 -# -# - name: Set up Python -# uses: actions/setup-python@v4 -# with: -# python-version: "3.10" -# cache: "pip" -# cache-dependency-path: | -# ${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt -# -# - name: Configure AWS Credentials -# uses: aws-actions/configure-aws-credentials@v2 -# with: -# role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} -# role-session-name: "gh-${{ env.ENVIRONMENT }}-integration-test" -# aws-region: "${{ env.AWS_DEFAULT_REGION }}" -# -# - name: Install python dependencies -# shell: bash -# working-directory: ${{ env.DIRECTORY }} -# run: | -# pip install -r requirements.txt -# python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" -# -# - name: Integration test -# shell: bash -# working-directory: ${{ env.DIRECTORY }} -# run: | -# pytest . -# -# run-playwright-checks: -# name: playwright end to end tests -# if: ${{ needs.define-environment.outputs.env_name }} -# runs-on: ubuntu-latest -# needs: [ deploy-veda-backend, deploy-veda-data-airflow] -# steps: -# - name: Checkout -# uses: actions/checkout@v4 -# with: -# repository: NASA-IMPACT/veda-config -# ref: add-playwright -# -# - name: Use Node.js 16 -# uses: actions/setup-node@v3 -# with: -# node-version: 16 -# -# - name: Run veda setup -# run: ./.veda/setup -# -# - name: Checkout generate_env script -# uses: actions/checkout@v4 -# with: -# sparse-checkout: | -# /scripts/generate_env_file.py -# sparse-checkout-cone-mode: false -# -# - name: Generate .env file from cdk.out -# shell: bash -# working-directory: ${{ env.DIRECTORY }} -# run: | -# pip install -r requirements.txt -# python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" -# -# - name: Load .env file -# id: dotenv -# uses: falti/dotenv-action@v1.1 -# -# - name: Create env.local file -# env: -# MAPBOX_TOKEN: ${{secrets.MAPBOX_TOKEN}} -# run: | -# touch .env.local -# echo "API_RASTER_ENDPOINT=${{steps.dotenv.outputs.VEDA_RASTER_URL}}" >> .env.local -# echo "API_STAC_ENDPOINT=${{steps.dotenv.outputs.VEDA_STAC_URL}}" >> .env.local -# -# echo "Testing deployment URLs" -# cat .env.local -# -# echo "MAPBOX_TOKEN="$MAPBOX_TOKEN" >> .env.local -# -# - name: Install Playwright Browsers -# run: yarn playwright install --with-deps -# -# - name: Playwright tests -# run: yarn test:e2e -# - uses: actions/upload-artifact@v3 -# if: always() -# with: -# name: playwright-report -# path: playwright-report/ -# retention-days: 30 + + deploy-veda-features-api: + name: Deploy VEDA features-api πŸ—ΊοΈ + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.environment }} + needs: [export-env-vars] + if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API && needs.export-env-vars.outputs.DEPLOY_FEATURES_API != 'false' + env: + DIRECTORY: veda-features-api-cdk + ENVIRONMENT: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: "true" + submodules: "false" + + - name: Checkout veda-features-api submodule + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_FEATURES_API_GIT_REF || 'main'}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-features-deployment" + aws-region: "us-west-2" + + - name: Run deployment + uses: "./veda-features-api-cdk/.github/actions/cdk-deploy" + id: deploy_features_stack + with: + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + dir: "${{ env.DIRECTORY }}" + script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + + - name: Get Features API Stack Name + id: get_features_api_stack + shell: bash + run: | + stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) + echo "features_api_stackname=$stack" >> $GITHUB_OUTPUT + + outputs: + features_api_stack_name: ${{ steps.get_features_api_stack.outputs.features_api_stackname }} + + deploy-veda-monitoring: + name: deploy VEDA monitoring πŸ‘οΈ + runs-on: ubuntu-latest + env: + DIRECTORY: veda-monitoring + AWS_REGION: "us-west-2" + ENVIRONMENT: ${{ github.event.inputs.environment }} + GH_PAT_CHECK: ${{ secrets.GH_PAT }} + needs: [deploy-veda-backend, export-env-vars] + if: needs.export-env-vars.outputs.DEPLOY_MONITORING != 'false' + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + if: ${{ env.GH_PAT_CHECK != '' }} + uses: actions/checkout@v4 + with: + lfs: "true" + submodules: "false" + + - name: Checkout veda-monitoring "submodule" + if: ${{ env.GH_PAT_CHECK != '' }} + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_MONITORING_GIT_REF || 'main' }} + token: ${{ secrets.GH_PAT }} + + - name: Configure AWS Credentials + if: ${{ env.GH_PAT_CHECK != '' }} + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-monitoring-deployment" + aws-region: "${{ env.AWS_REGION }}" + + - name: Run deployment + if: ${{ env.GH_PAT_CHECK != '' }} + uses: "./veda-monitoring/.github/actions/cdk-deploy" + with: + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + dir: "${{ env.DIRECTORY }}" + script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + + test-deployment: + name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ + runs-on: ubuntu-latest + needs: [ deploy-veda-backend ] + env: + DIRECTORY: integration_test + ENVIRONMENT: ${{ github.event.inputs.environment }} + AWS_DEFAULT_REGION: us-west-2 + environment: ${{ github.event.inputs.environment }} + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + cache: "pip" + cache-dependency-path: | + ${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-integration-test" + aws-region: "${{ env.AWS_DEFAULT_REGION }}" + + - name: Install python dependencies + shell: bash + working-directory: ${{ env.DIRECTORY }} + run: | + pip install -r requirements.txt + python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" + + - name: Integration test + shell: bash + working-directory: ${{ env.DIRECTORY }} + run: | + pytest . + + run-playwright-checks: + name: playwright end to end tests + if: ${{ needs.define-environment.outputs.env_name }} + runs-on: ubuntu-latest + needs: [ deploy-veda-backend, deploy-veda-data-airflow] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + repository: NASA-IMPACT/veda-config + ref: add-playwright + + - name: Use Node.js 16 + uses: actions/setup-node@v3 + with: + node-version: 16 + + - name: Run veda setup + run: ./.veda/setup + + - name: Checkout generate_env script + uses: actions/checkout@v4 + with: + sparse-checkout: | + /scripts/generate_env_file.py + sparse-checkout-cone-mode: false + + - name: Generate .env file from cdk.out + shell: bash + working-directory: ${{ env.DIRECTORY }} + run: | + pip install -r requirements.txt + python "${{ github.workspace }}/scripts/generate_env_file.py" --secret-id ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} --stack-names "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" + + - name: Load .env file + id: dotenv + uses: falti/dotenv-action@v1.1 + + - name: Create env.local file + env: + MAPBOX_TOKEN: ${{secrets.MAPBOX_TOKEN}} + run: | + touch .env.local + echo "API_RASTER_ENDPOINT=${{steps.dotenv.outputs.VEDA_RASTER_URL}}" >> .env.local + echo "API_STAC_ENDPOINT=${{steps.dotenv.outputs.VEDA_STAC_URL}}" >> .env.local + + echo "Testing deployment URLs" + cat .env.local + + echo "MAPBOX_TOKEN="$MAPBOX_TOKEN" >> .env.local + + - name: Install Playwright Browsers + run: yarn playwright install --with-deps + + - name: Playwright tests + run: yarn test:e2e + - uses: actions/upload-artifact@v3 + if: always() + with: + name: playwright-report + path: playwright-report/ + retention-days: 30 \ No newline at end of file From 65f049f708111a7559717fb5eee77a57b5bdf091 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Tue, 13 Aug 2024 13:24:04 -0500 Subject: [PATCH 077/210] Add SM2A to GHG DEV --- .github/workflows/cicd.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 70e464c..3e3ad40 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -195,13 +195,6 @@ jobs: backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" - - name: Get Workflows API Endpoint - id: get_workflows_api_endpoint - shell: bash - run: | - workflows_api_value=$(cat ${HOME}/terraform_outputs.json) - echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT - - name: Run SM2A deployment id: deploy_sm2a if: ${{ vars.DEPLOY_SM2A }} = "true" @@ -212,16 +205,18 @@ jobs: dir: "${{ env.DIRECTORY }}/sm2a" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - - name: Get Workflows SM2A API Endpoint - id: get_sm2a_workflows_api_endpoint + - name: Get Workflows API Endpoint + id: get_workflows_api_endpoint shell: bash run: | + workflows_api_value=$(cat ${HOME}/terraform_outputs.json) workflows_sm2a_api_value=$(cat ${HOME}/output_sm2a_workflows_endpoint.json 2>/dev/null | tr -d '"' | tr -d '\n' ) + echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT echo "workflows_sm2a_api_value=${workflows_sm2a_api_value:-NA}" >> $GITHUB_OUTPUT outputs: workflows_api_endpoint: ${{ steps.get_workflows_api_endpoint.outputs.workflows_api_endpoint }} - workflows_sm2a_api_value: ${{ steps.get_sm2a_workflows_api_endpoint.outputs.workflows_sm2a_api_value }} + workflows_sm2a_api_value: ${{ steps.get_workflows_api_endpoint.outputs.workflows_sm2a_api_value }} deploy-veda-features-api: name: Deploy VEDA features-api πŸ—ΊοΈ From 186705a273d1fd1cbe82ea28aa3e3de1e8560472 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Tue, 13 Aug 2024 13:27:36 -0500 Subject: [PATCH 078/210] Add newline --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 3e3ad40..eedd3d7 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -413,4 +413,4 @@ jobs: with: name: playwright-report path: playwright-report/ - retention-days: 30 \ No newline at end of file + retention-days: From df633dac45a8496a3b8581bcfe84f9e03a7f1ebe Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Wed, 14 Aug 2024 09:44:05 -0500 Subject: [PATCH 079/210] Add README to deploy SM2A --- .github/workflows/cicd.yml | 4 ++-- README.md | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index eedd3d7..43f465e 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -216,7 +216,7 @@ jobs: outputs: workflows_api_endpoint: ${{ steps.get_workflows_api_endpoint.outputs.workflows_api_endpoint }} - workflows_sm2a_api_value: ${{ steps.get_workflows_api_endpoint.outputs.workflows_sm2a_api_value }} + workflows_sm2a_api_value: ${{ steps.get_sm2a_workflows_api_endpoint.outputs.workflows_sm2a_api_value }} deploy-veda-features-api: name: Deploy VEDA features-api πŸ—ΊοΈ @@ -413,4 +413,4 @@ jobs: with: name: playwright-report path: playwright-report/ - retention-days: + retention-days: 30 diff --git a/README.md b/README.md index 2af3acd..96e772c 100644 --- a/README.md +++ b/README.md @@ -68,3 +68,9 @@ TF_VAR_gh_app_client_secret=****** TF_VAR_gh_team_name=****** TF_VAR_subdomain=****** ``` +##### Github variables +Add these variables to Github environment variables +```bash +DEPLOY_SM2A=true +SM2A_ENVS_DEPLOYMENT_SECRET_NAME= +``` From 6f0bec4f4143854de5a72088447588ec6e39693a Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Thu, 15 Aug 2024 13:21:34 -0500 Subject: [PATCH 080/210] Add SM2A to VEDA Deploy --- .github/workflows/cicd.yml | 50 +++++++++++++++++++++++++++++++++----- README.md | 1 + 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 43f465e..d97b468 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -195,9 +195,50 @@ jobs: backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" + - name: Get Workflows API Endpoint + id: get_workflows_api_endpoint + shell: bash + run: | + workflows_api_value=$(cat ${HOME}/terraform_outputs.json) + echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT + + outputs: + workflows_api_endpoint: ${{ steps.get_workflows_api_endpoint.outputs.workflows_api_endpoint }} + + deploy-veda-data-airflow-sm2a: + name: deploy VEDA data airflow SM2A πŸ›Έ + runs-on: ubuntu-latest + env: + DIRECTORY: veda-data-airflow + AWS_REGION: "us-west-2" + ENVIRONMENT: ${{ github.event.inputs.environment }} + needs: [ export-env-vars ] + if: ${{ vars.DEPLOY_SM2A }} = "true" + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: "true" + submodules: "false" + + - name: Checkout veda-data-airflow submodule + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_SM2A_DATA_AIRFLOW_GIT_REF || 'main'}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-airflow-sm2a-deployment" + aws-region: "${{ env.AWS_REGION }}" + - name: Run SM2A deployment id: deploy_sm2a - if: ${{ vars.DEPLOY_SM2A }} = "true" uses: "./veda-data-airflow/.github/actions/terraform-deploy-sm2a" with: env-file: ".env" @@ -205,17 +246,14 @@ jobs: dir: "${{ env.DIRECTORY }}/sm2a" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - - name: Get Workflows API Endpoint - id: get_workflows_api_endpoint + - name: Get SM2A Workflows API Endpoint + id: get_sm2a_workflows_api_endpoint shell: bash run: | - workflows_api_value=$(cat ${HOME}/terraform_outputs.json) workflows_sm2a_api_value=$(cat ${HOME}/output_sm2a_workflows_endpoint.json 2>/dev/null | tr -d '"' | tr -d '\n' ) - echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT echo "workflows_sm2a_api_value=${workflows_sm2a_api_value:-NA}" >> $GITHUB_OUTPUT outputs: - workflows_api_endpoint: ${{ steps.get_workflows_api_endpoint.outputs.workflows_api_endpoint }} workflows_sm2a_api_value: ${{ steps.get_sm2a_workflows_api_endpoint.outputs.workflows_sm2a_api_value }} deploy-veda-features-api: diff --git a/README.md b/README.md index 96e772c..b0a30e5 100644 --- a/README.md +++ b/README.md @@ -73,4 +73,5 @@ Add these variables to Github environment variables ```bash DEPLOY_SM2A=true SM2A_ENVS_DEPLOYMENT_SECRET_NAME= +VEDA_SM2A_DATA_AIRFLOW_GIT_REF= ``` From 14e6352cb0cffb13bd6f588445ce94383a6f9467 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Mon, 19 Aug 2024 16:54:00 -0500 Subject: [PATCH 081/210] Fix condition to deploy SM2A --- .github/workflows/cicd.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index d97b468..51aa567 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -28,6 +28,7 @@ jobs: DEPLOY_MONITORING: ${{ steps.init.outputs.DEPLOY_MONITORING }} DEPLOY_AUTH: ${{ steps.init.outputs.DEPLOY_AUTH }} DEPLOY_BACKEND: ${{ steps.init.outputs.DEPLOY_BACKEND }} + DEPLOY_SM2A: ${{ steps.init.outputs.DEPLOY_SM2A }} steps: - name: Environment variables to output @@ -38,6 +39,7 @@ jobs: echo "DEPLOY_MONITORING=${{ vars.DEPLOY_MONITORING }}" >> $GITHUB_OUTPUT echo "DEPLOY_AUTH=${{ vars.DEPLOY_AUTH }}" >> $GITHUB_OUTPUT echo "DEPLOY_BACKEND=${{ vars.DEPLOY_BACKEND }}" >> $GITHUB_OUTPUT + echo "DEPLOY_SM2A=${{ vars.DEPLOY_SM2A }}" >> $GITHUB_OUTPUT deploy-veda-auth: name: Deploy VEDA auth πŸ” @@ -213,7 +215,8 @@ jobs: AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} needs: [ export-env-vars ] - if: ${{ vars.DEPLOY_SM2A }} = "true" + needs: [export-env-vars] + if: needs.export-env-vars.outputs.DEPLOY_SM2A && needs.export-env-vars.outputs.DEPLOY_SM2A != 'false' environment: ${{ github.event.inputs.environment }} steps: From f89d9446324c7692ee2dd22bad85570e5fa09d4e Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Mon, 19 Aug 2024 17:01:52 -0500 Subject: [PATCH 082/210] Remove duplicate needs --- .github/workflows/cicd.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 51aa567..8bcf5bd 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -215,7 +215,6 @@ jobs: AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} needs: [ export-env-vars ] - needs: [export-env-vars] if: needs.export-env-vars.outputs.DEPLOY_SM2A && needs.export-env-vars.outputs.DEPLOY_SM2A != 'false' environment: ${{ github.event.inputs.environment }} From 789485dafbb4a7367965f52f6bf06235c674f27d Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Wed, 21 Aug 2024 15:39:04 -0500 Subject: [PATCH 083/210] Remove the monitoring deployment dependency on veda backend --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 8bcf5bd..c9824d2 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -315,7 +315,7 @@ jobs: AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} - needs: [deploy-veda-backend, export-env-vars] + needs: [export-env-vars] if: needs.export-env-vars.outputs.DEPLOY_MONITORING != 'false' environment: ${{ github.event.inputs.environment }} From 63bc876cee675242fef1c118bc643e866b77b4ed Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Mon, 9 Sep 2024 10:53:19 -0500 Subject: [PATCH 084/210] Adding github actions inputes to control the deployments of VEDA modules --- .github/workflows/cicd.yml | 76 +++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index c9824d2..9c1c67f 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -11,6 +11,41 @@ on: type: environment required: true description: Environment to deploy to + DEPLOY_AUTH: + type: boolean + required: true + default: true + description: DEPLOY_AUTH + DEPLOY_BACKEND: + type: boolean + required: true + default: true + description: DEPLOY_BACKEND + DEPLOY_DATA_AIRFLOW: + type: boolean + required: true + default: true + description: DEPLOY_DATA_AIRFLOW + DEPLOY_FEATURES_API: + type: boolean + required: true + default: false + description: DEPLOY_FEATURES_API + DEPLOY_ROUTES: + type: boolean + required: true + default: false + description: DEPLOY_ROUTES + DEPLOY_SM2A: + type: boolean + required: true + default: true + description: DEPLOY_SM2A + DEPLOY_MONITORING: + type: boolean + required: true + default: false + description: DEPLOY_MONITORING concurrency: group: ${{ inputs.environment }} @@ -19,27 +54,6 @@ concurrency: run-name: Deploy to ${{ inputs.environment }} environment by @${{ github.actor }} jobs: - export-env-vars: - runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment }} - outputs: - DEPLOY_FEATURES_API: ${{ steps.init.outputs.DEPLOY_FEATURES_API }} - DEPLOY_DATA_AIRFLOW: ${{ steps.init.outputs.DEPLOY_DATA_AIRFLOW }} - DEPLOY_MONITORING: ${{ steps.init.outputs.DEPLOY_MONITORING }} - DEPLOY_AUTH: ${{ steps.init.outputs.DEPLOY_AUTH }} - DEPLOY_BACKEND: ${{ steps.init.outputs.DEPLOY_BACKEND }} - DEPLOY_SM2A: ${{ steps.init.outputs.DEPLOY_SM2A }} - - steps: - - name: Environment variables to output - id: init - run: | - echo "DEPLOY_FEATURES_API=${{ vars.DEPLOY_FEATURES_API }}" >> $GITHUB_OUTPUT - echo "DEPLOY_DATA_AIRFLOW=${{ vars.DEPLOY_DATA_AIRFLOW }}" >> $GITHUB_OUTPUT - echo "DEPLOY_MONITORING=${{ vars.DEPLOY_MONITORING }}" >> $GITHUB_OUTPUT - echo "DEPLOY_AUTH=${{ vars.DEPLOY_AUTH }}" >> $GITHUB_OUTPUT - echo "DEPLOY_BACKEND=${{ vars.DEPLOY_BACKEND }}" >> $GITHUB_OUTPUT - echo "DEPLOY_SM2A=${{ vars.DEPLOY_SM2A }}" >> $GITHUB_OUTPUT deploy-veda-auth: name: Deploy VEDA auth πŸ” @@ -48,8 +62,7 @@ jobs: DIRECTORY: veda-auth ENVIRONMENT: ${{ github.event.inputs.environment }} environment: ${{ github.event.inputs.environment }} - needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_AUTH != 'false' + if: ${{ github.event.inputs.DEPLOY_AUTH == 'true' }} steps: - name: Checkout @@ -95,8 +108,8 @@ jobs: env: DIRECTORY: veda-backend ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [deploy-veda-auth, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_BACKEND != 'false' + needs: [deploy-veda-auth] + if: ${{ github.event.inputs.DEPLOY_BACKEND == 'true' }} environment: ${{ github.event.inputs.environment }} steps: @@ -162,8 +175,8 @@ jobs: DIRECTORY: veda-data-airflow AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [deploy-veda-auth, deploy-veda-backend, export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW && needs.export-env-vars.outputs.DEPLOY_DATA_AIRFLOW != 'false' + needs: [deploy-veda-auth, deploy-veda-backend] + if: ${{ github.event.inputs.DEPLOY_DATA_AIRFLOW == 'true' }} environment: ${{ github.event.inputs.environment }} steps: @@ -214,8 +227,7 @@ jobs: DIRECTORY: veda-data-airflow AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [ export-env-vars ] - if: needs.export-env-vars.outputs.DEPLOY_SM2A && needs.export-env-vars.outputs.DEPLOY_SM2A != 'false' + if: ${{ github.event.inputs.DEPLOY_SM2A == 'true' }} environment: ${{ github.event.inputs.environment }} steps: @@ -262,8 +274,7 @@ jobs: name: Deploy VEDA features-api πŸ—ΊοΈ runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment }} - needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_FEATURES_API && needs.export-env-vars.outputs.DEPLOY_FEATURES_API != 'false' + if: ${{ github.event.inputs.DEPLOY_FEATURES_API == 'true' }} env: DIRECTORY: veda-features-api-cdk ENVIRONMENT: ${{ github.event.inputs.environment }} @@ -315,8 +326,7 @@ jobs: AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} - needs: [export-env-vars] - if: needs.export-env-vars.outputs.DEPLOY_MONITORING != 'false' + if: ${{ github.event.inputs.DEPLOY_MONITORING == 'true' }} environment: ${{ github.event.inputs.environment }} steps: From 9788cb076a0fcd5347b0fe880d3318862d20474b Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Mon, 9 Sep 2024 13:45:47 -0500 Subject: [PATCH 085/210] Only make auth and backend default deployment to true --- .github/workflows/cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 9c1c67f..67c10b0 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -24,7 +24,7 @@ on: DEPLOY_DATA_AIRFLOW: type: boolean required: true - default: true + default: false description: DEPLOY_DATA_AIRFLOW DEPLOY_FEATURES_API: type: boolean @@ -39,7 +39,7 @@ on: DEPLOY_SM2A: type: boolean required: true - default: true + default: false description: DEPLOY_SM2A DEPLOY_MONITORING: type: boolean From a604eb78f5710ab12c031a25458ca8c6366b3b88 Mon Sep 17 00:00:00 2001 From: Ciaran Sweet Date: Tue, 13 Aug 2024 10:31:49 +0100 Subject: [PATCH 086/210] feat: add veda-routes deployment --- .github/workflows/cicd.yml | 70 +++++++++++++++++++++++++++- .gitmodules | 3 ++ scripts/update_secret_with_inputs.py | 61 ++++++++++++++++++++++++ veda-routes | 1 + 4 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 scripts/update_secret_with_inputs.py create mode 160000 veda-routes diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 67c10b0..d2e0a6d 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -362,10 +362,78 @@ jobs: dir: "${{ env.DIRECTORY }}" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + deploy-veda-routes: + name: deploy VEDA routes πŸ”€ + runs-on: ubuntu-latest + env: + DIRECTORY: veda-routes + AWS_REGION: "us-west-2" + ENVIRONMENT: ${{ github.event.inputs.environment }} + needs: [ deploy-veda-backend, deploy-veda-data-airflow ] + if: ${{ github.event.inputs.DEPLOY_ROUTES == 'true' }} + environment: ${{ github.event.inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: "true" + submodules: "false" + + - name: Checkout veda-routes "submodule" + uses: actions/checkout@v4 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_ROUTES_GIT_REF || 'main' }} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-routes-deployment" + aws-region: "${{ env.AWS_REGION }}" + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Setup python cache + uses: actions/cache@v4 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt') }} + + - name: Install python dependencies + working-directory: ${{ env.DIRECTORY }} + shell: bash + run: | + pip install -r requirements.txt + + - name: Update Veda Routes secrets with dependency outputs + shell: bash + run: | + python3 "${{ github.workspace }}/scripts/update_secret_with_inputs.py" \ + --secret-id ${{ vars.DEPLOYMENT_ENV_ROUTES_SECRET_NAME }} \ + --prefix=VEDA_ \ + --raster_api_url=${{ needs.deploy-veda-backend.outputs.raster_api_url }} \ + --ingest_api_url=${{ needs.deploy-veda-backend.outputs.ingest_api_url }} \ + --stac_api_url=${{ needs.deploy-veda-backend.outputs.stac_api_url }} \ + --stac_browser_bucket_name=${{ needs.deploy-veda-backend.outputs.stac_browser_bucket_name }} \ + --workflows_api_url=${{ needs.deploy-veda-data-airflow.outputs.workflows_api_endpoint }} \ + + - name: Run deployment + uses: "./veda-routes/.github/actions/cdk-deploy" + with: + env_aws_secret_name: ${{ env.DEPLOYMENT_ENV_ROUTES_SECRET_NAME }} + dir: "${{ env.DIRECTORY }}" + + test-deployment: name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ runs-on: ubuntu-latest - needs: [ deploy-veda-backend ] + needs: [ deploy-veda-routes ] env: DIRECTORY: integration_test ENVIRONMENT: ${{ github.event.inputs.environment }} diff --git a/.gitmodules b/.gitmodules index 7e3e264..bd4080f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "veda-features-api-cdk"] path = veda-features-api-cdk url = git@github.com:NASA-IMPACT/veda-features-api-cdk.git +[submodule "veda-routes"] + path = veda-routes + url = git@github.com:NASA-IMPACT/veda-routes.git diff --git a/scripts/update_secret_with_inputs.py b/scripts/update_secret_with_inputs.py new file mode 100644 index 0000000..abd7f72 --- /dev/null +++ b/scripts/update_secret_with_inputs.py @@ -0,0 +1,61 @@ +import argparse +import json +import re + +import boto3 + +_ARG_REGEX = r'--(.*)=(.*)' + + +def update_secret_with_inputs(): + parser = argparse.ArgumentParser( + description=""" + **WARNING** This is destructive, if an input is provided that's name already exists in + the AWS Secret, it will be overridden. + + --- + + Takes in N inputs in the form --input-name=value and inserts them into the provided + AWS SecretsManager secret. If a prefix is provided, it will be appended to the input name + + This assumes that the SecretString value is a stringified JSON object + + For example, with no prefix, an input of --my-secret-item=hello will be inserted as: + + MY_SECRET_ITEM=hello + + Whereas with a prefix of MY_PREFIX, an input of --my-secret-item=hello will be inserted as: + + MY_PREFIX_MY_SECRET_ITEM=hello + """, + formatter_class=argparse.RawTextHelpFormatter + ) + + parser.add_argument("--prefix", default="", required=False, help="Optional prefix to append to inputs names") + parser.add_argument("--secret-id", required=True, help="The ARN or Name of the AWS SecretsManager secret to update") + + known_args, unknown_args = parser.parse_known_args() + + if not (secret_id := known_args.secret_id): + raise Exception("An AWS SecretsManager secret id is required") + + values_to_add_to_secret = {} + + for arg in unknown_args: + if match := re.match(_ARG_REGEX, arg): + secret_entry_name = known_args.prefix.upper() + match.group(1).upper().replace("-", '_') + secret_entry_value = match.group(2) + values_to_add_to_secret[secret_entry_name] = secret_entry_value + + secrets_manager_client = boto3.client("secretsmanager") + secret = secrets_manager_client.get_secret_value(SecretId=secret_id) + secret_value = json.loads(secret["SecretString"]) + + for k, v in values_to_add_to_secret.items(): + secret_value[k] = v + + secrets_manager_client.put_secret_value(SecretId=secret_id, SecretString=json.dumps(secret_value)) + + +if __name__ == "__main__": + update_secret_with_inputs() diff --git a/veda-routes b/veda-routes new file mode 160000 index 0000000..8ac256a --- /dev/null +++ b/veda-routes @@ -0,0 +1 @@ +Subproject commit 8ac256af750d0d0011d9c9a86349bba9b5efa8a8 From b12cf9d8f296f1f0fbbfd76d0d6cd703aee93053 Mon Sep 17 00:00:00 2001 From: Ciaran Sweet Date: Tue, 10 Sep 2024 15:23:55 +0100 Subject: [PATCH 087/210] fix: main -> dev --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index d2e0a6d..68e2815 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -385,7 +385,7 @@ jobs: with: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} - ref: ${{ vars.VEDA_ROUTES_GIT_REF || 'main' }} + ref: ${{ vars.VEDA_ROUTES_GIT_REF || 'dev' }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 From 0981e49b7df0566fdecb6129764ea57b011dcd12 Mon Sep 17 00:00:00 2001 From: Ciaran Sweet Date: Tue, 10 Sep 2024 15:54:22 +0100 Subject: [PATCH 088/210] fix: attempt to use github token --- .github/workflows/cicd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 68e2815..10372bc 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -386,6 +386,7 @@ jobs: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} ref: ${{ vars.VEDA_ROUTES_GIT_REF || 'dev' }} + token: ${{ github.token }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 From 73adc5eab357e55b98a43fa455c55b0474e1b92c Mon Sep 17 00:00:00 2001 From: Ciaran Sweet Date: Tue, 10 Sep 2024 16:30:52 +0100 Subject: [PATCH 089/210] fix: swap back to PAT --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 10372bc..1b72a53 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -386,7 +386,7 @@ jobs: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} ref: ${{ vars.VEDA_ROUTES_GIT_REF || 'dev' }} - token: ${{ github.token }} + token: ${{ secrets.GH_PAT }} - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v2 From f1af6cea0c1650c5767cc9274d2b531e5c13d05e Mon Sep 17 00:00:00 2001 From: Ciaran Sweet Date: Tue, 10 Sep 2024 16:50:34 +0100 Subject: [PATCH 090/210] fix: install deploy requirements too --- .github/workflows/cicd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 1b72a53..d43eaf2 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -410,6 +410,7 @@ jobs: working-directory: ${{ env.DIRECTORY }} shell: bash run: | + pip install -r ../requirements.txt pip install -r requirements.txt - name: Update Veda Routes secrets with dependency outputs From c81a9cd2244ac7f27f9d281f6ad230ce9e7c745f Mon Sep 17 00:00:00 2001 From: Ciaran Sweet Date: Tue, 10 Sep 2024 17:15:09 +0100 Subject: [PATCH 091/210] fix: correct variables --- .github/workflows/cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index d43eaf2..e73d835 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -159,7 +159,7 @@ jobs: echo "stac_api_url=$stac_api_url" >> $GITHUB_OUTPUT stack_browser_bucket_name=$(jq '.[keys_unsorted[0]].stacbrowserbucketname' ${HOME}/cdk-outputs.json) - echo "stack_browser_bucket_name=stack_browser_bucket_name" >> $GITHUB_OUTPUT + echo "stack_browser_bucket_name=$stack_browser_bucket_name" >> $GITHUB_OUTPUT outputs: backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} @@ -428,7 +428,7 @@ jobs: - name: Run deployment uses: "./veda-routes/.github/actions/cdk-deploy" with: - env_aws_secret_name: ${{ env.DEPLOYMENT_ENV_ROUTES_SECRET_NAME }} + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_ROUTES_SECRET_NAME }} dir: "${{ env.DIRECTORY }}" From c07de34a369bd5c71f688b2911f792fc6e651548 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Tue, 10 Sep 2024 12:24:25 -0500 Subject: [PATCH 092/210] Adding manual approvers for production env --- .github/workflows/cicd.yml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 67c10b0..aabd489 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -3,6 +3,7 @@ name: CI/CD πŸš€ permissions: id-token: write contents: read + issues: write on: workflow_dispatch: @@ -54,10 +55,27 @@ concurrency: run-name: Deploy to ${{ inputs.environment }} environment by @${{ github.actor }} jobs: + check-environment: + runs-on: ubuntu-latest + name: Got ${{ github.event.inputs.environment }} + steps: + - name: Validation + uses: trstringer/manual-approval@v1 + if: ${{ github.event.inputs.environment == 'ghgc-mcp-production-blue' }} + timeout-minutes: 60 # The approver will have 1 hour to approve this request + # Why 1h? Because GitHub App tokens expire after 1 hour which implies duration + # for the approval cannot exceed 60 minutes or the job will fail due to bad credentials + with: + secret: ${{ secrets.GITHUB_TOKEN }} + approvers: amarouane-ABDELHAK,slesaad + minimum-approvals: 1 + issue-title: "Deploying to GHG production" + issue-body: "Please approve or deny the deployment" deploy-veda-auth: name: Deploy VEDA auth πŸ” runs-on: ubuntu-latest + needs: check-environment env: DIRECTORY: veda-auth ENVIRONMENT: ${{ github.event.inputs.environment }} @@ -108,7 +126,7 @@ jobs: env: DIRECTORY: veda-backend ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [deploy-veda-auth] + needs: [check-environment, deploy-veda-auth] if: ${{ github.event.inputs.DEPLOY_BACKEND == 'true' }} environment: ${{ github.event.inputs.environment }} @@ -175,7 +193,7 @@ jobs: DIRECTORY: veda-data-airflow AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [deploy-veda-auth, deploy-veda-backend] + needs: [check-environment, deploy-veda-auth, deploy-veda-backend] if: ${{ github.event.inputs.DEPLOY_DATA_AIRFLOW == 'true' }} environment: ${{ github.event.inputs.environment }} @@ -223,6 +241,7 @@ jobs: deploy-veda-data-airflow-sm2a: name: deploy VEDA data airflow SM2A πŸ›Έ runs-on: ubuntu-latest + needs: check-environment env: DIRECTORY: veda-data-airflow AWS_REGION: "us-west-2" @@ -273,6 +292,7 @@ jobs: deploy-veda-features-api: name: Deploy VEDA features-api πŸ—ΊοΈ runs-on: ubuntu-latest + needs: check-environment environment: ${{ github.event.inputs.environment }} if: ${{ github.event.inputs.DEPLOY_FEATURES_API == 'true' }} env: @@ -321,6 +341,7 @@ jobs: deploy-veda-monitoring: name: deploy VEDA monitoring πŸ‘οΈ runs-on: ubuntu-latest + needs: check-environment env: DIRECTORY: veda-monitoring AWS_REGION: "us-west-2" From ea6c9868b5cb179d44556fbdf7fd58a6a5dd211e Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Tue, 10 Sep 2024 12:29:58 -0500 Subject: [PATCH 093/210] Adding VEDA production environement --- .github/workflows/cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index aabd489..abe502a 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -61,13 +61,13 @@ jobs: steps: - name: Validation uses: trstringer/manual-approval@v1 - if: ${{ github.event.inputs.environment == 'ghgc-mcp-production-blue' }} + if: ${{ github.event.inputs.environment == 'ghgc-mcp-production-blue' || github.event.inputs.environment == 'mcp-prod' }} timeout-minutes: 60 # The approver will have 1 hour to approve this request # Why 1h? Because GitHub App tokens expire after 1 hour which implies duration # for the approval cannot exceed 60 minutes or the job will fail due to bad credentials with: secret: ${{ secrets.GITHUB_TOKEN }} - approvers: amarouane-ABDELHAK,slesaad + approvers: amarouane-ABDELHAK,slesaad,anayeaye,smohiudd,ciaransweet,botanical,ividito minimum-approvals: 1 issue-title: "Deploying to GHG production" issue-body: "Please approve or deny the deployment" From 087a83207be96359757ca494ba6a590da0a52625 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Tue, 10 Sep 2024 12:32:29 -0500 Subject: [PATCH 094/210] Adding VEDA production environement --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index abe502a..c13436f 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -69,7 +69,7 @@ jobs: secret: ${{ secrets.GITHUB_TOKEN }} approvers: amarouane-ABDELHAK,slesaad,anayeaye,smohiudd,ciaransweet,botanical,ividito minimum-approvals: 1 - issue-title: "Deploying to GHG production" + issue-title: "Deploying to ${{ github.event.inputs.environment }}" issue-body: "Please approve or deny the deployment" deploy-veda-auth: From ca0e945686ee6e41f554d27b4b2b1e2784aeee60 Mon Sep 17 00:00:00 2001 From: Ciaran Sweet Date: Tue, 10 Sep 2024 17:45:22 +0100 Subject: [PATCH 095/210] fix: typo in stac bucket name --- .github/workflows/cicd.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index e73d835..13b5cd3 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -158,15 +158,15 @@ jobs: stac_api_url=$(jq '.[keys_unsorted[0]].stacapiurl' ${HOME}/cdk-outputs.json) echo "stac_api_url=$stac_api_url" >> $GITHUB_OUTPUT - stack_browser_bucket_name=$(jq '.[keys_unsorted[0]].stacbrowserbucketname' ${HOME}/cdk-outputs.json) - echo "stack_browser_bucket_name=$stack_browser_bucket_name" >> $GITHUB_OUTPUT + stac_browser_bucket_name=$(jq '.[keys_unsorted[0]].stacbrowserbucketname' ${HOME}/cdk-outputs.json) + echo "stac_browser_bucket_name=$stac_browser_bucket_name" >> $GITHUB_OUTPUT outputs: backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} raster_api_url: ${{ steps.get_backend_stack.outputs.raster_api_url }} ingest_api_url: ${{ steps.get_backend_stack.outputs.ingest_api_url }} stac_api_url: ${{ steps.get_backend_stack.outputs.stac_api_url }} - stack_browser_bucket_name: ${{ steps.get_backend_stack.outputs.stack_browser_bucket_name }} + stac_browser_bucket_name: ${{ steps.get_backend_stack.outputs.stac_browser_bucket_name }} deploy-veda-data-airflow: name: deploy VEDA data airflow πŸƒ From e6e4a92e2ddfb3ac79d75370213eab8e320f4d01 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Thu, 12 Sep 2024 14:15:12 -0500 Subject: [PATCH 096/210] Make dispatch call the deployment --- .github/workflows/{cicd.yml => deploy.yml} | 89 +++++++++------------- .github/workflows/dispatch.yml | 83 ++++++++++++++++++++ 2 files changed, 118 insertions(+), 54 deletions(-) rename .github/workflows/{cicd.yml => deploy.yml} (84%) create mode 100644 .github/workflows/dispatch.yml diff --git a/.github/workflows/cicd.yml b/.github/workflows/deploy.yml similarity index 84% rename from .github/workflows/cicd.yml rename to .github/workflows/deploy.yml index c13436f..693f846 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/deploy.yml @@ -1,49 +1,50 @@ -name: CI/CD πŸš€ +name: Deploy πŸš€ permissions: id-token: write contents: read issues: write + on: - workflow_dispatch: + workflow_call: inputs: environment: - type: environment + type: string required: true description: Environment to deploy to DEPLOY_AUTH: - type: boolean + type: string required: true default: true description: DEPLOY_AUTH DEPLOY_BACKEND: - type: boolean + type: string required: true default: true description: DEPLOY_BACKEND DEPLOY_DATA_AIRFLOW: - type: boolean + type: string required: true - default: false + default: true description: DEPLOY_DATA_AIRFLOW DEPLOY_FEATURES_API: - type: boolean + type: string required: true default: false description: DEPLOY_FEATURES_API DEPLOY_ROUTES: - type: boolean + type: string required: true default: false description: DEPLOY_ROUTES DEPLOY_SM2A: - type: boolean + type: string required: true - default: false + default: true description: DEPLOY_SM2A DEPLOY_MONITORING: - type: boolean + type: string required: true default: false description: DEPLOY_MONITORING @@ -55,32 +56,15 @@ concurrency: run-name: Deploy to ${{ inputs.environment }} environment by @${{ github.actor }} jobs: - check-environment: - runs-on: ubuntu-latest - name: Got ${{ github.event.inputs.environment }} - steps: - - name: Validation - uses: trstringer/manual-approval@v1 - if: ${{ github.event.inputs.environment == 'ghgc-mcp-production-blue' || github.event.inputs.environment == 'mcp-prod' }} - timeout-minutes: 60 # The approver will have 1 hour to approve this request - # Why 1h? Because GitHub App tokens expire after 1 hour which implies duration - # for the approval cannot exceed 60 minutes or the job will fail due to bad credentials - with: - secret: ${{ secrets.GITHUB_TOKEN }} - approvers: amarouane-ABDELHAK,slesaad,anayeaye,smohiudd,ciaransweet,botanical,ividito - minimum-approvals: 1 - issue-title: "Deploying to ${{ github.event.inputs.environment }}" - issue-body: "Please approve or deny the deployment" deploy-veda-auth: name: Deploy VEDA auth πŸ” runs-on: ubuntu-latest - needs: check-environment env: DIRECTORY: veda-auth - ENVIRONMENT: ${{ github.event.inputs.environment }} - environment: ${{ github.event.inputs.environment }} - if: ${{ github.event.inputs.DEPLOY_AUTH == 'true' }} + ENVIRONMENT: ${{ inputs.environment }} + environment: ${{ inputs.environment }} + if: ${{ inputs.DEPLOY_AUTH == 'true' }} steps: - name: Checkout @@ -125,10 +109,10 @@ jobs: runs-on: ubuntu-latest env: DIRECTORY: veda-backend - ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [check-environment, deploy-veda-auth] - if: ${{ github.event.inputs.DEPLOY_BACKEND == 'true' }} - environment: ${{ github.event.inputs.environment }} + ENVIRONMENT: ${{ inputs.environment }} + needs: [deploy-veda-auth] + if: ${{ inputs.DEPLOY_BACKEND == 'true' }} + environment: ${{ inputs.environment }} steps: - name: Checkout @@ -192,10 +176,10 @@ jobs: env: DIRECTORY: veda-data-airflow AWS_REGION: "us-west-2" - ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [check-environment, deploy-veda-auth, deploy-veda-backend] - if: ${{ github.event.inputs.DEPLOY_DATA_AIRFLOW == 'true' }} - environment: ${{ github.event.inputs.environment }} + ENVIRONMENT: ${{ inputs.environment }} + needs: [deploy-veda-auth, deploy-veda-backend] + if: ${{ inputs.DEPLOY_DATA_AIRFLOW == 'true' }} + environment: ${{ inputs.environment }} steps: - name: Checkout @@ -241,13 +225,12 @@ jobs: deploy-veda-data-airflow-sm2a: name: deploy VEDA data airflow SM2A πŸ›Έ runs-on: ubuntu-latest - needs: check-environment env: DIRECTORY: veda-data-airflow AWS_REGION: "us-west-2" - ENVIRONMENT: ${{ github.event.inputs.environment }} - if: ${{ github.event.inputs.DEPLOY_SM2A == 'true' }} - environment: ${{ github.event.inputs.environment }} + ENVIRONMENT: ${{ inputs.environment }} + if: ${{ inputs.DEPLOY_SM2A == 'true' }} + environment: ${{ inputs.environment }} steps: - name: Checkout @@ -292,12 +275,11 @@ jobs: deploy-veda-features-api: name: Deploy VEDA features-api πŸ—ΊοΈ runs-on: ubuntu-latest - needs: check-environment - environment: ${{ github.event.inputs.environment }} - if: ${{ github.event.inputs.DEPLOY_FEATURES_API == 'true' }} + environment: ${{ inputs.environment }} + if: ${{ inputs.DEPLOY_FEATURES_API == 'true' }} env: DIRECTORY: veda-features-api-cdk - ENVIRONMENT: ${{ github.event.inputs.environment }} + ENVIRONMENT: ${{ inputs.environment }} steps: - name: Checkout @@ -341,14 +323,13 @@ jobs: deploy-veda-monitoring: name: deploy VEDA monitoring πŸ‘οΈ runs-on: ubuntu-latest - needs: check-environment env: DIRECTORY: veda-monitoring AWS_REGION: "us-west-2" - ENVIRONMENT: ${{ github.event.inputs.environment }} + ENVIRONMENT: ${{ inputs.environment }} GH_PAT_CHECK: ${{ secrets.GH_PAT }} - if: ${{ github.event.inputs.DEPLOY_MONITORING == 'true' }} - environment: ${{ github.event.inputs.environment }} + if: ${{ inputs.DEPLOY_MONITORING == 'true' }} + environment: ${{ inputs.environment }} steps: - name: Checkout @@ -389,9 +370,9 @@ jobs: needs: [ deploy-veda-backend ] env: DIRECTORY: integration_test - ENVIRONMENT: ${{ github.event.inputs.environment }} + ENVIRONMENT: ${{ inputs.environment }} AWS_DEFAULT_REGION: us-west-2 - environment: ${{ github.event.inputs.environment }} + environment: ${{ inputs.environment }} steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml new file mode 100644 index 0000000..3966d67 --- /dev/null +++ b/.github/workflows/dispatch.yml @@ -0,0 +1,83 @@ +name: Dispatch β›Ÿ + +permissions: + id-token: write + contents: read + issues: write + +on: + workflow_dispatch: + inputs: + environment: + type: environment + required: true + description: Environment to deploy to + DEPLOY_AUTH: + type: boolean + required: true + default: true + description: DEPLOY_AUTH + DEPLOY_BACKEND: + type: boolean + required: true + default: true + description: DEPLOY_BACKEND + DEPLOY_DATA_AIRFLOW: + type: boolean + required: true + default: false + description: DEPLOY_DATA_AIRFLOW + DEPLOY_FEATURES_API: + type: boolean + required: true + default: false + description: DEPLOY_FEATURES_API + DEPLOY_ROUTES: + type: boolean + required: true + default: false + description: DEPLOY_ROUTES + DEPLOY_SM2A: + type: boolean + required: true + default: false + description: DEPLOY_SM2A + DEPLOY_MONITORING: + type: boolean + required: true + default: false + description: DEPLOY_MONITORING + + +jobs: + check-environment: + runs-on: ubuntu-latest + name: Got ${{ github.event.inputs.environment }} + steps: + - name: Validation + uses: trstringer/manual-approval@v1 + if: ${{ github.event.inputs.environment == 'ghgc-mcp-production-blue' || github.event.inputs.environment == 'mcp-prod' }} + timeout-minutes: 60 # The approver will have 1 hour to approve this request + # Why 1h? Because GitHub App tokens expire after 1 hour which implies duration + # for the approval cannot exceed 60 minutes or the job will fail due to bad credentials + with: + secret: ${{ secrets.GITHUB_TOKEN }} + approvers: amarouane-ABDELHAK,slesaad,anayeaye,smohiudd,ciaransweet,botanical,ividito,stephenkilbourn + minimum-approvals: 1 + issue-title: "Deploying to ${{ github.event.inputs.environment }}" + issue-body: "Please approve or deny the deployment" + + deploy-veda-components: + name: Deploy VEDA Components + uses: "./.github/workflows/deploy.yml" + needs: check-environment + with: + environment: ${{ github.event.inputs.environment }} + DEPLOY_AUTH: ${{ github.event.inputs.DEPLOY_AUTH}} + DEPLOY_BACKEND: ${{ github.event.inputs.DEPLOY_BACKEND }} + DEPLOY_DATA_AIRFLOW: ${{ github.event.inputs.DEPLOY_DATA_AIRFLOW }} + DEPLOY_FEATURES_API: ${{ github.event.inputs.DEPLOY_FEATURES_API }} + DEPLOY_ROUTES: ${{ github.event.inputs.DEPLOY_ROUTES }} + DEPLOY_SM2A: ${{ github.event.inputs.DEPLOY_SM2A }} + DEPLOY_MONITORING: ${{ github.event.inputs.DEPLOY_MONITORING }} + secrets: inherit From 936d191ba73eb29105a95bd577095d819d7a2b21 Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:00:04 -0700 Subject: [PATCH 097/210] fix: test test-deployment dependency --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 13b5cd3..4c486c1 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -435,7 +435,7 @@ jobs: test-deployment: name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ runs-on: ubuntu-latest - needs: [ deploy-veda-routes ] + needs: [ deploy-veda-backend, deploy-veda-routes ] env: DIRECTORY: integration_test ENVIRONMENT: ${{ github.event.inputs.environment }} From 161726f9360ad4d6d2aca85cd6aecf8b04a3a2d9 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 28 Oct 2024 13:02:33 -0600 Subject: [PATCH 098/210] fix: expand deploy run name to describe environment and components --- .github/workflows/deploy.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 506bfcf..48825c0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,7 +53,10 @@ concurrency: group: ${{ inputs.environment }} cancel-in-progress: true -run-name: Deploy to ${{ inputs.environment }} environment by @${{ github.actor }} +run-name: Deploy to ${{ inputs.environment }} by @${{ github.actor }} \ + BACKEND=${{ inputs.DEPLOY_BACKEND }} AIRFLOW=${{ inputs.DEPLOY_DATA_AIRFLOW }} \ + FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} \ + SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} jobs: From e421a079ad115e82bcd76fae813451bf9e551c34 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 28 Oct 2024 13:42:15 -0600 Subject: [PATCH 099/210] move annotation to dispatch --- .github/workflows/deploy.yml | 5 +---- .github/workflows/dispatch.yml | 4 ++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 48825c0..506bfcf 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,10 +53,7 @@ concurrency: group: ${{ inputs.environment }} cancel-in-progress: true -run-name: Deploy to ${{ inputs.environment }} by @${{ github.actor }} \ - BACKEND=${{ inputs.DEPLOY_BACKEND }} AIRFLOW=${{ inputs.DEPLOY_DATA_AIRFLOW }} \ - FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} \ - SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} +run-name: Deploy to ${{ inputs.environment }} environment by @${{ github.actor }} jobs: diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 3966d67..c0dda67 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -48,6 +48,10 @@ on: default: false description: DEPLOY_MONITORING +run-name: Dispatch to ${{ inputs.environment }} by @${{ github.actor }} \ + BACKEND=${{ inputs.DEPLOY_BACKEND }} AIRFLOW=${{ inputs.DEPLOY_DATA_AIRFLOW }} \ + FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} \ + SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} jobs: check-environment: From c67c80a69878c5e713b34cf17d56adef20f3f925 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 28 Oct 2024 13:46:57 -0600 Subject: [PATCH 100/210] formatting --- .github/workflows/dispatch.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index c0dda67..8879430 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -48,10 +48,7 @@ on: default: false description: DEPLOY_MONITORING -run-name: Dispatch to ${{ inputs.environment }} by @${{ github.actor }} \ - BACKEND=${{ inputs.DEPLOY_BACKEND }} AIRFLOW=${{ inputs.DEPLOY_DATA_AIRFLOW }} \ - FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} \ - SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} +run-name: Dispatch to ${{ inputs.environment }} by @${{ github.actor }} BACKEND=${{ inputs.DEPLOY_BACKEND }} AIRFLOW=${{ inputs.DEPLOY_DATA_AIRFLOW }}FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} jobs: check-environment: From d8273cb13a650d6e272f7f9dd90d558d036f4d7e Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 28 Oct 2024 13:52:11 -0600 Subject: [PATCH 101/210] formatting --- .github/workflows/dispatch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 8879430..d1ddb1e 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -48,7 +48,7 @@ on: default: false description: DEPLOY_MONITORING -run-name: Dispatch to ${{ inputs.environment }} by @${{ github.actor }} BACKEND=${{ inputs.DEPLOY_BACKEND }} AIRFLOW=${{ inputs.DEPLOY_DATA_AIRFLOW }}FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} +run-name: Dispatch to ${{ inputs.environment }} by @${{ github.actor }} BACKEND=${{ inputs.DEPLOY_BACKEND }} AIRFLOW=${{ inputs.DEPLOY_DATA_AIRFLOW }}FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} β›Ÿ jobs: check-environment: From c45c1a99b1b69ea9be42fac1ee17619f22b562c1 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 28 Oct 2024 14:03:25 -0600 Subject: [PATCH 102/210] formatting --- .github/workflows/dispatch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index d1ddb1e..4ee8168 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -48,7 +48,7 @@ on: default: false description: DEPLOY_MONITORING -run-name: Dispatch to ${{ inputs.environment }} by @${{ github.actor }} BACKEND=${{ inputs.DEPLOY_BACKEND }} AIRFLOW=${{ inputs.DEPLOY_DATA_AIRFLOW }}FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} β›Ÿ +run-name: Dispatch to ${{ inputs.environment }} by @${{ github.actor }} BACKEND=${{ inputs.DEPLOY_BACKEND }} AIRFLOW=${{ inputs.DEPLOY_DATA_AIRFLOW }} FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} β›Ÿ jobs: check-environment: From f64dbc3a688081437baeedbb9e0159b005f76669 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 28 Oct 2024 17:13:17 -0600 Subject: [PATCH 103/210] do not pre select environment --- .github/workflows/dispatch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 4ee8168..55a45d0 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -12,6 +12,7 @@ on: type: environment required: true description: Environment to deploy to + default: none DEPLOY_AUTH: type: boolean required: true From dca56615fc50fcf3ffee86686e86a2d23c016b2b Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Tue, 29 Oct 2024 17:40:31 -0600 Subject: [PATCH 104/210] reset dispatch env --- .github/workflows/dispatch.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 55a45d0..4ee8168 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -12,7 +12,6 @@ on: type: environment required: true description: Environment to deploy to - default: none DEPLOY_AUTH: type: boolean required: true From 20ed1535cf37bd7a7cf41230d5491a46ef84a447 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 31 Oct 2024 13:42:53 -0600 Subject: [PATCH 105/210] add integration test for next links behind a proxy and do not test apigw urls when default is disabled --- integration_test/test_api_health.py | 70 +++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/integration_test/test_api_health.py b/integration_test/test_api_health.py index 5e2e316..6f35cb9 100644 --- a/integration_test/test_api_health.py +++ b/integration_test/test_api_health.py @@ -4,17 +4,22 @@ load_dotenv() +def _get_link(obj: dict, rel: str) -> str: + """get rel link from a stac object""" + return next((l for l in obj.get("links") if l["rel"]==rel), None) def test_stac_url_returns_200(): base_url = os.getenv("VEDA_STAC_URL") stac_root_path = os.getenv("VEDA_STAC_ROOT_PATH") custom_host = os.getenv("VEDA_CUSTOM_HOST", None) + disable_default_apigw = os.getenv("VEDA_DISABLE_DEFAULT_APIGW_ENDPOINT", False) health_endpoint = "_mgmt/ping" - url = f"{base_url}{health_endpoint}" - print(f"Checking APIGW stac-api {url=}") - response = requests.get(url) - assert response.status_code == 200 + if not disable_default_apigw: + url = f"{base_url}{health_endpoint}" # APIGW base url includes trailing / + print(f"Checking APIGW stac-api {url=}") + response = requests.get(url) + assert response.status_code == 200 if custom_host: url = f"https://{custom_host}/{stac_root_path.rstrip('/')}/{health_endpoint}" @@ -28,15 +33,64 @@ def test_raster_url_returns_200(): base_url = os.getenv("VEDA_RASTER_URL") raster_root_path = os.getenv("VEDA_RASTER_ROOT_PATH") custom_host = os.getenv("VEDA_CUSTOM_HOST", None) + disable_default_apigw = os.getenv("VEDA_DISABLE_DEFAULT_APIGW_ENDPOINT", False) health_endpoint = "healthz" - url = os.path.join(base_url, health_endpoint) - print(f"Checking APIGW raster-api {url=}") - response = requests.get(url) - assert response.status_code == 200 + if not disable_default_apigw: + url = os.path.join(base_url, health_endpoint) + print(f"Checking APIGW raster-api {url=}") + response = requests.get(url) + assert response.status_code == 200 if custom_host: url = f"https://{custom_host}/{raster_root_path.rstrip('/')}/{health_endpoint}" print(f"Checking custom host raster-api {url=}") response = requests.get(url) assert response.status_code == 200 + +def test_stac_item_next_link_returns_200(): + base_url = os.getenv("VEDA_STAC_URL") + stac_root_path = os.getenv("VEDA_STAC_ROOT_PATH") + custom_host = os.getenv("VEDA_CUSTOM_HOST", None) + disable_default_apigw = os.getenv("VEDA_DISABLE_DEFAULT_APIGW_ENDPOINT", False) + collections_endpoint = "collections" + + if not disable_default_apigw: + url = f"{base_url}/{collections_endpoint}" + print(f"Checking APIGW stac-api {url=}") + response = requests.get(url) + assert response.status_code == 200 + + if custom_host: + url = f"https://{custom_host}/{stac_root_path.rstrip('/')}/{collections_endpoint}" + print(f"Checking custom host stac-api {url=}") + response = requests.get(url) + assert response.status_code == 200 + + # Walk check root path propogation through dynamic links when using custom host + collections = response.json().get("collections") + next_links_untested = True + + while next_links_untested: + for collection in collections: + + # All collections should have a dynamicaly generateed items link, even if no items exist + items_link = _get_link(collection, "items") + assert items_link + items_url = items_link.get("href") + assert items_url + items_response = requests.get(items_url) + assert items_response.status_code == 200 + items_json = items_response.json() + features = items_json.get("features") + + # The default page size is 10 + if len(features) >= 10: + items_next_link = _get_link(items_json, "next") + assert items_next_link + next_url = items_next_link.get("href") + assert next_url + next_response = requests.get(next_url) + assert next_response.status_code == 200 + next_links_untested = False + break \ No newline at end of file From 9b6cdad794fc2fe2e216709c3c16f1ae90b6087e Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 31 Oct 2024 13:50:23 -0600 Subject: [PATCH 106/210] improve print statement in test --- integration_test/test_api_health.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_test/test_api_health.py b/integration_test/test_api_health.py index 6f35cb9..68bf419 100644 --- a/integration_test/test_api_health.py +++ b/integration_test/test_api_health.py @@ -63,7 +63,7 @@ def test_stac_item_next_link_returns_200(): if custom_host: url = f"https://{custom_host}/{stac_root_path.rstrip('/')}/{collections_endpoint}" - print(f"Checking custom host stac-api {url=}") + print(f"Checking links for custom host stac-api {url=}") response = requests.get(url) assert response.status_code == 200 From e01a41c8210d5c87633e27215f06eb035b2052c5 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 31 Oct 2024 15:53:48 -0600 Subject: [PATCH 107/210] typo and new line --- integration_test/test_api_health.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_test/test_api_health.py b/integration_test/test_api_health.py index 68bf419..6b03bbf 100644 --- a/integration_test/test_api_health.py +++ b/integration_test/test_api_health.py @@ -67,7 +67,7 @@ def test_stac_item_next_link_returns_200(): response = requests.get(url) assert response.status_code == 200 - # Walk check root path propogation through dynamic links when using custom host + # Walk check root path propagation through dynamic links when using custom host collections = response.json().get("collections") next_links_untested = True @@ -93,4 +93,4 @@ def test_stac_item_next_link_returns_200(): next_response = requests.get(next_url) assert next_response.status_code == 200 next_links_untested = False - break \ No newline at end of file + break From 35e2fba2b6d7e2888ece136fc9fa93e0c14eadbe Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Fri, 1 Nov 2024 08:28:17 -0600 Subject: [PATCH 108/210] Update integration_test/test_api_health.py Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- integration_test/test_api_health.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_test/test_api_health.py b/integration_test/test_api_health.py index 6b03bbf..9530ba7 100644 --- a/integration_test/test_api_health.py +++ b/integration_test/test_api_health.py @@ -74,7 +74,7 @@ def test_stac_item_next_link_returns_200(): while next_links_untested: for collection in collections: - # All collections should have a dynamicaly generateed items link, even if no items exist + # All collections should have a dynamicaly generated items link, even if no items exist items_link = _get_link(collection, "items") assert items_link items_url = items_link.get("href") From 909f070fe132298daef3d4fb0df0a36c7170f19d Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Tue, 12 Nov 2024 13:41:15 -0700 Subject: [PATCH 109/210] ci: remove mwaa deployment job and downstream job requirements --- .github/workflows/deploy.yml | 64 ++-------------------------------- .github/workflows/dispatch.yml | 8 +---- 2 files changed, 4 insertions(+), 68 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 506bfcf..8aa5492 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -23,11 +23,6 @@ on: required: true default: true description: DEPLOY_BACKEND - DEPLOY_DATA_AIRFLOW: - type: string - required: true - default: true - description: DEPLOY_DATA_AIRFLOW DEPLOY_FEATURES_API: type: string required: true @@ -170,58 +165,6 @@ jobs: stac_api_url: ${{ steps.get_backend_stack.outputs.stac_api_url }} stac_browser_bucket_name: ${{ steps.get_backend_stack.outputs.stac_browser_bucket_name }} - deploy-veda-data-airflow: - name: deploy VEDA data airflow πŸƒ - runs-on: ubuntu-latest - env: - DIRECTORY: veda-data-airflow - AWS_REGION: "us-west-2" - ENVIRONMENT: ${{ inputs.environment }} - needs: [deploy-veda-auth, deploy-veda-backend] - if: ${{ inputs.DEPLOY_DATA_AIRFLOW == 'true' }} - environment: ${{ inputs.environment }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - lfs: "true" - submodules: "false" - - - name: Checkout veda-data-airflow submodule - uses: actions/checkout@v4 - with: - repository: "NASA-IMPACT/${{ env.DIRECTORY }}" - path: ${{ env.DIRECTORY }} - ref: ${{ vars.VEDA_DATA_AIRFLOW_GIT_REF || 'main'}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "gh-${{ env.ENVIRONMENT }}-airflow-deployment" - aws-region: "${{ env.AWS_REGION }}" - - - name: Run MWAA deployment - uses: "./veda-data-airflow/.github/actions/terraform-deploy" - with: - env-file: ".env" - env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - dir: "${{ env.DIRECTORY }}" - script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - backend_stack_name: "${{ needs.deploy-veda-backend.outputs.backend_stack_name }}" - auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" - - - name: Get Workflows API Endpoint - id: get_workflows_api_endpoint - shell: bash - run: | - workflows_api_value=$(cat ${HOME}/terraform_outputs.json) - echo "workflows_api_endpoint=$workflows_api_value" >> $GITHUB_OUTPUT - - outputs: - workflows_api_endpoint: ${{ steps.get_workflows_api_endpoint.outputs.workflows_api_endpoint }} - deploy-veda-data-airflow-sm2a: name: deploy VEDA data airflow SM2A πŸ›Έ runs-on: ubuntu-latest @@ -371,7 +314,7 @@ jobs: DIRECTORY: veda-routes AWS_REGION: "us-west-2" ENVIRONMENT: ${{ github.event.inputs.environment }} - needs: [ deploy-veda-backend, deploy-veda-data-airflow ] + needs: [ deploy-veda-backend ] if: ${{ github.event.inputs.DEPLOY_ROUTES == 'true' }} environment: ${{ github.event.inputs.environment }} @@ -424,8 +367,7 @@ jobs: --raster_api_url=${{ needs.deploy-veda-backend.outputs.raster_api_url }} \ --ingest_api_url=${{ needs.deploy-veda-backend.outputs.ingest_api_url }} \ --stac_api_url=${{ needs.deploy-veda-backend.outputs.stac_api_url }} \ - --stac_browser_bucket_name=${{ needs.deploy-veda-backend.outputs.stac_browser_bucket_name }} \ - --workflows_api_url=${{ needs.deploy-veda-data-airflow.outputs.workflows_api_endpoint }} \ + --stac_browser_bucket_name=${{ needs.deploy-veda-backend.outputs.stac_browser_bucket_name }} - name: Run deployment uses: "./veda-routes/.github/actions/cdk-deploy" @@ -478,7 +420,7 @@ jobs: name: playwright end to end tests if: ${{ needs.define-environment.outputs.env_name }} runs-on: ubuntu-latest - needs: [ deploy-veda-backend, deploy-veda-data-airflow] + needs: [ deploy-veda-backend ] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 4ee8168..46f1f16 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -22,11 +22,6 @@ on: required: true default: true description: DEPLOY_BACKEND - DEPLOY_DATA_AIRFLOW: - type: boolean - required: true - default: false - description: DEPLOY_DATA_AIRFLOW DEPLOY_FEATURES_API: type: boolean required: true @@ -48,7 +43,7 @@ on: default: false description: DEPLOY_MONITORING -run-name: Dispatch to ${{ inputs.environment }} by @${{ github.actor }} BACKEND=${{ inputs.DEPLOY_BACKEND }} AIRFLOW=${{ inputs.DEPLOY_DATA_AIRFLOW }} FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} β›Ÿ +run-name: Dispatch to ${{ inputs.environment }} BACKEND=${{ inputs.DEPLOY_BACKEND }} AIRFLOW=${{ inputs.DEPLOY_DATA_AIRFLOW }} FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} β›Ÿ jobs: check-environment: @@ -76,7 +71,6 @@ jobs: environment: ${{ github.event.inputs.environment }} DEPLOY_AUTH: ${{ github.event.inputs.DEPLOY_AUTH}} DEPLOY_BACKEND: ${{ github.event.inputs.DEPLOY_BACKEND }} - DEPLOY_DATA_AIRFLOW: ${{ github.event.inputs.DEPLOY_DATA_AIRFLOW }} DEPLOY_FEATURES_API: ${{ github.event.inputs.DEPLOY_FEATURES_API }} DEPLOY_ROUTES: ${{ github.event.inputs.DEPLOY_ROUTES }} DEPLOY_SM2A: ${{ github.event.inputs.DEPLOY_SM2A }} From 0fccab5d421cf6f8c1ed3d1d173454397dfebe1c Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Tue, 12 Nov 2024 14:39:26 -0700 Subject: [PATCH 110/210] fix: upgrade configure aws credentials --- .github/workflows/deploy.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8aa5492..e7398d1 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -76,7 +76,7 @@ jobs: ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" @@ -124,7 +124,7 @@ jobs: ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main'}} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-backend-deployment" @@ -239,7 +239,7 @@ jobs: ref: ${{ vars.VEDA_FEATURES_API_GIT_REF || 'main'}} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-features-deployment" @@ -293,7 +293,7 @@ jobs: - name: Configure AWS Credentials if: ${{ env.GH_PAT_CHECK != '' }} - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-monitoring-deployment" @@ -334,7 +334,7 @@ jobs: token: ${{ secrets.GH_PAT }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-routes-deployment" @@ -397,7 +397,7 @@ jobs: ${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-integration-test" From 046c888ba20b328c3f67fbdf1faf00eaef67d17f Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Tue, 12 Nov 2024 14:45:36 -0700 Subject: [PATCH 111/210] run integration tests even without routes --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e7398d1..aad8fdc 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -379,7 +379,7 @@ jobs: test-deployment: name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ runs-on: ubuntu-latest - needs: [ deploy-veda-backend, deploy-veda-routes ] + needs: [ deploy-veda-backend ] env: DIRECTORY: integration_test ENVIRONMENT: ${{ inputs.environment }} From 201ad26fece28f742ca197ec242d285ba91fd852 Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Wed, 13 Nov 2024 13:41:54 -0600 Subject: [PATCH 112/210] Add veda secret name condition --- .github/workflows/deploy.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index aad8fdc..18e0e6e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -87,14 +87,15 @@ jobs: uses: "./veda-auth/.github/actions/cdk-deploy" with: dir: "${{ env.DIRECTORY }}" - env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + env_aws_secret_name: ${{ vars.VEDA_AUTH_DEPLOYMENT_SECRET || vars.DEPLOYMENT_ENV_SECRET_NAME }} - name: Get Auth Stack Name id: get_auth_stack shell: bash run: | - stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) - echo "auth_stackname=$stack" >> $GITHUB_OUTPUT + echo "This is secret name ${env_aws_secret_name}" + # stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) + # echo "auth_stackname=$stack" >> $GITHUB_OUTPUT outputs: auth_stack_name: ${{ steps.get_auth_stack.outputs.auth_stackname }} From a98812a4420546aa0ce552f77f2994bfc8b6cfdb Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Wed, 13 Nov 2024 13:46:07 -0600 Subject: [PATCH 113/210] Add veda secret name condition --- .github/workflows/deploy.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 18e0e6e..afc2310 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -93,9 +93,8 @@ jobs: id: get_auth_stack shell: bash run: | - echo "This is secret name ${env_aws_secret_name}" - # stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) - # echo "auth_stackname=$stack" >> $GITHUB_OUTPUT + stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) + echo "auth_stackname=$stack" >> $GITHUB_OUTPUT outputs: auth_stack_name: ${{ steps.get_auth_stack.outputs.auth_stackname }} From 0d6c20045c512fdb2892b091be2e710abd1d13e4 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Wed, 13 Nov 2024 15:43:02 -0700 Subject: [PATCH 114/210] fix dispatch name components summary --- .github/workflows/dispatch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 46f1f16..66eeea6 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -43,7 +43,7 @@ on: default: false description: DEPLOY_MONITORING -run-name: Dispatch to ${{ inputs.environment }} BACKEND=${{ inputs.DEPLOY_BACKEND }} AIRFLOW=${{ inputs.DEPLOY_DATA_AIRFLOW }} FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} β›Ÿ +run-name: Dispatch to ${{ inputs.environment }} AUTH=${{ inputs.DEPLOY_AUTH }} BACKEND=${{ inputs.DEPLOY_BACKEND }} FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} β›Ÿ jobs: check-environment: From 312e57344d101d82e62724a89f5b3f4624df268d Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Fri, 20 Dec 2024 10:34:37 -0700 Subject: [PATCH 115/210] fix: do not pre select env in dispatch --- .github/workflows/deploy.yml | 1 + .github/workflows/dispatch.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index afc2310..619b7e6 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,6 +13,7 @@ on: type: string required: true description: Environment to deploy to + default: none DEPLOY_AUTH: type: string required: true diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 66eeea6..16dc8e1 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -12,6 +12,7 @@ on: type: environment required: true description: Environment to deploy to + default: none DEPLOY_AUTH: type: boolean required: true From 07fc20ef8a1c48bfedafe152a937814fcac4e7d9 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Fri, 20 Dec 2024 10:38:47 -0700 Subject: [PATCH 116/210] fix: do not pre select env in dispatch --- .github/workflows/deploy.yml | 2 +- .github/workflows/dispatch.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 619b7e6..84f85e9 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,7 +13,7 @@ on: type: string required: true description: Environment to deploy to - default: none + default: "select-env" DEPLOY_AUTH: type: string required: true diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 16dc8e1..fcf40e4 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -12,7 +12,7 @@ on: type: environment required: true description: Environment to deploy to - default: none + default: "select-env" DEPLOY_AUTH: type: boolean required: true From 6546aaab792bc1d6e1dfa1e224898198596f7c1f Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Fri, 20 Dec 2024 10:44:23 -0700 Subject: [PATCH 117/210] fix: do not pre select env in dispatch --- .github/workflows/deploy.yml | 2 +- .github/workflows/dispatch.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 84f85e9..f8ac77c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,7 +13,7 @@ on: type: string required: true description: Environment to deploy to - default: "select-env" + default: '' DEPLOY_AUTH: type: string required: true diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index fcf40e4..77c4a04 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -12,7 +12,7 @@ on: type: environment required: true description: Environment to deploy to - default: "select-env" + default: '' DEPLOY_AUTH: type: boolean required: true From c8402a7e9e82069ca9bea19adbd78821eedc6785 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Fri, 20 Dec 2024 10:47:01 -0700 Subject: [PATCH 118/210] fix: revert env default --- .github/workflows/deploy.yml | 1 - .github/workflows/dispatch.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f8ac77c..afc2310 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,7 +13,6 @@ on: type: string required: true description: Environment to deploy to - default: '' DEPLOY_AUTH: type: string required: true diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 77c4a04..66eeea6 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -12,7 +12,6 @@ on: type: environment required: true description: Environment to deploy to - default: '' DEPLOY_AUTH: type: boolean required: true From c72a5dfbd7c27f92ac0538038997a7d75e7c7408 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 30 Jan 2025 11:04:19 -0700 Subject: [PATCH 119/210] docs: draft steps to add new component --- README.md | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b0a30e5..2000782 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ VEDA_RASTER_PATH_PREFIX=***** ``` -#### AWS Secrets Requirements For SM2A +#### AWS Secrets Requirements for SM2A ```bash AIRFLOW_UID=****** PREFIX=****** @@ -75,3 +75,64 @@ DEPLOY_SM2A=true SM2A_ENVS_DEPLOYMENT_SECRET_NAME= VEDA_SM2A_DATA_AIRFLOW_GIT_REF= ``` + +# Add New Components +## Overview +1. [Add deployment action to component github repository](#add-deployment-action-to-component-github-repository) +2. [Add component submodule to veda-deploy](#add-component-submodule-to-veda-deploy) +3. [Store `.env` configuration in AWS Secrets Manager](#store-env-configuration-in-aws-secrets-manager) +4. [Extend composite dispatched deployment action with an optional component job that uses the component submodule and environment secret](#extend-composite-dispatched-deployment-action) +5. [Add new component release version and environment secret name to veda-deploy environment(s)](#add-new-component-release-version-and-environment-secret-name-to-veda-deploy-environments) +6. [Configure domain and custom routes](#configure-domain-and-custom-routes) + +## Add deployment action to component github repository +Dispatches from veda-deploy are composed of deployment actions imported from github submodules. The management of all configuration, testing, and deployment concerns is managed within the component's github repository (not in veda-deploy). + +Create a new `cdk-deploy/action.yml` in the component project's repository. On a dispatch the configured release version of the project will be checked out and executed on the veda-deploy github runner. + +To keep the components modular, each action should include all necessary steps for deployment including Python and Node setup steps. While veda-deploy uses the same runner to deploy all components, it should not be assumed that the runner already has all needed installations and environment configuration from other components (unless a dependency is configured for the job using needs: {upstream-job-name}). + +### Examples +- Veda-auth [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-auth/blob/main/.github/actions/cdk-deploy/action.yml) provides a simple example of adding configuration from an Aws Secrets Manager secret and running cdk deploy for an imported submodule. +- Veda-backend [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/actions/cdk-deploy/action.yml) contains logic to run tests before deploying comonents. +- This [CICD workflow in veda-backend](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/workflows/cicd.yml) demonstrates importing the cdk-deploy/action on a merge event to test the deployment in a dev enviornment. + +## Add component submodule to veda-deploy +Add your component submodule to [.gitmodules](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.gitmodules). Submodules are checked out on the github runner when your component is deployed. + +``` +[submodule "my-project"] + path = my-project + url = git@github.com:NASA-IMPACT/my-project.git +``` + +## Store `.env` configuration in AWS Secrets Manager +Custom configurations like RDS instance size as well as AWS environment specific configuration like VPC ID and a Permission Boundary Policy Name should be added to a key-value secret that will be loaded into the github runner environment by your action. This secret should be stored in the target AWS account where the component will be deployed. + +> **Note:** +> 1. For higher security environments, a permissions boundary policy needs to be identified. +> 2. The qualifier of the CDK Toolkit bootstrapped for the target environment must be provided if not using the default toolkit. + +### Sample environmnt variables +``` +VPC_ID=****** +PERMISSIONS_BOUNDARY_POLICY_NAME=****** +STAGE=****** +BOOTSTRAP_QUALIFIER=****** +``` + +## Extend composite dispatched deployment action + +1. Add a [dispatch flag in .github/workflows/dispatch.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/dispatch.yml#L66) component you are adding. As in `DEPLOY_MY_COMONENT: ${{ github.event.inputs.DEPLOY_MY_COMPONENT }}` +2. Update the [dispatch message](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/dispatch.yml#L46) to include your component. Eventually this will get too long and will need some thought but it is currently helpful to filter the actions and identify specific dispatches. +3. Transfer the above dispatch information to the [.github/workflows/deploy.yml workflow_call](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/deploy.yml#L10). The deploy action is called after the environment is set by the dispatch and for production environments is only executed after the dispatch has been approved by a maintainer. +4. Add a new [named job to deploy.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/deploy.yml#L218) that checks the condition the deployment condition for the component and, when true, checks out the deployment action from the component's github repository and passes in any relevant information like the configuration environment secret name. + +## Add new component release version and environment secret name to veda-deploy environment(s) +Adding new deployment environments requires admin permissions for this veda-deploy repository. New environments are added by entering project settings and selecting `Environments` from the code and automation menu. The environment naming convention is `-`, i.e. `smce-staging`. As more environments are added this convention will need to be updated. + +In the Environment variables for the instance you are dispatching your component to, add a new variable with the github reference to the release you want to deploy. + +`MY_COMPONENT_GIT_REF=v1.0` + +## Configure domain and custom routes \ No newline at end of file From 2db8a84424753bdbba5da721dc3a7f8810ac426a Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 30 Jan 2025 13:58:10 -0700 Subject: [PATCH 120/210] Update README.md Co-authored-by: Aimee Barciauskas --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2000782..669a26a 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ BOOTSTRAP_QUALIFIER=****** ## Extend composite dispatched deployment action -1. Add a [dispatch flag in .github/workflows/dispatch.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/dispatch.yml#L66) component you are adding. As in `DEPLOY_MY_COMONENT: ${{ github.event.inputs.DEPLOY_MY_COMPONENT }}` +1. Add a [dispatch flag in .github/workflows/dispatch.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/dispatch.yml#L66) component you are adding. As in `DEPLOY_MY_COMPONENT: ${{ github.event.inputs.DEPLOY_MY_COMPONENT }}` 2. Update the [dispatch message](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/dispatch.yml#L46) to include your component. Eventually this will get too long and will need some thought but it is currently helpful to filter the actions and identify specific dispatches. 3. Transfer the above dispatch information to the [.github/workflows/deploy.yml workflow_call](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/deploy.yml#L10). The deploy action is called after the environment is set by the dispatch and for production environments is only executed after the dispatch has been approved by a maintainer. 4. Add a new [named job to deploy.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/deploy.yml#L218) that checks the condition the deployment condition for the component and, when true, checks out the deployment action from the component's github repository and passes in any relevant information like the configuration environment secret name. From 1d8df47ebdb3196ad2a88662958cf01f4a8d0e77 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 30 Jan 2025 13:58:18 -0700 Subject: [PATCH 121/210] Update README.md Co-authored-by: Aimee Barciauskas --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 669a26a..81b6950 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ To keep the components modular, each action should include all necessary steps f ### Examples - Veda-auth [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-auth/blob/main/.github/actions/cdk-deploy/action.yml) provides a simple example of adding configuration from an Aws Secrets Manager secret and running cdk deploy for an imported submodule. -- Veda-backend [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/actions/cdk-deploy/action.yml) contains logic to run tests before deploying comonents. +- Veda-backend [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/actions/cdk-deploy/action.yml) contains logic to run tests before deploying components. - This [CICD workflow in veda-backend](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/workflows/cicd.yml) demonstrates importing the cdk-deploy/action on a merge event to test the deployment in a dev enviornment. ## Add component submodule to veda-deploy From 8ee6fa2e2efe0f15322ff76a7d0c13e0547d8cec Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Fri, 31 Jan 2025 09:28:22 -0700 Subject: [PATCH 122/210] address feedback --- README.md | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 81b6950..f95e088 100644 --- a/README.md +++ b/README.md @@ -77,13 +77,16 @@ VEDA_SM2A_DATA_AIRFLOW_GIT_REF= ``` # Add New Components +> [!IMPORTANT] +> This section is intended to expand an existing configured environment, see [How to Deploy](#how-to-deploy) to start from scratch. Please read the full overview before starting, some steps overlap. + ## Overview -1. [Add deployment action to component github repository](#add-deployment-action-to-component-github-repository) -2. [Add component submodule to veda-deploy](#add-component-submodule-to-veda-deploy) -3. [Store `.env` configuration in AWS Secrets Manager](#store-env-configuration-in-aws-secrets-manager) -4. [Extend composite dispatched deployment action with an optional component job that uses the component submodule and environment secret](#extend-composite-dispatched-deployment-action) -5. [Add new component release version and environment secret name to veda-deploy environment(s)](#add-new-component-release-version-and-environment-secret-name-to-veda-deploy-environments) -6. [Configure domain and custom routes](#configure-domain-and-custom-routes) +- [Add deployment action to component github repository](#add-deployment-action-to-component-github-repository) +- [Store `.env` configuration in AWS Secrets Manager](#store-env-configuration-in-aws-secrets-manager) +- [Add component submodule to veda-deploy](#add-component-submodule-to-veda-deploy) +- [Extend composite dispatched deployment action with an optional component job that uses the component submodule and environment secret](#extend-composite-dispatched-deployment-action) +- [Add new component release version and environment secret name to veda-deploy environment(s)](#add-new-component-release-version-and-environment-secret-name-to-veda-deploy-environments) +- [Configure domain and custom routes](#configure-domain-and-custom-routes) ## Add deployment action to component github repository Dispatches from veda-deploy are composed of deployment actions imported from github submodules. The management of all configuration, testing, and deployment concerns is managed within the component's github repository (not in veda-deploy). @@ -92,24 +95,18 @@ Create a new `cdk-deploy/action.yml` in the component project's repository. On a To keep the components modular, each action should include all necessary steps for deployment including Python and Node setup steps. While veda-deploy uses the same runner to deploy all components, it should not be assumed that the runner already has all needed installations and environment configuration from other components (unless a dependency is configured for the job using needs: {upstream-job-name}). +> [!TIP] +> Most deployments require [custom environment configuration](#store-env-configuration-in-aws-secrets-manager) that can be retrieved from the AWS Secrets Manager for the deployment. See [veda-backend/scripts/get-env.sh](https://github.com/NASA-IMPACT/veda-backend/blob/develop/scripts/get-env.sh) for an example environmennt configuration utility. + ### Examples - Veda-auth [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-auth/blob/main/.github/actions/cdk-deploy/action.yml) provides a simple example of adding configuration from an Aws Secrets Manager secret and running cdk deploy for an imported submodule. - Veda-backend [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/actions/cdk-deploy/action.yml) contains logic to run tests before deploying components. - This [CICD workflow in veda-backend](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/workflows/cicd.yml) demonstrates importing the cdk-deploy/action on a merge event to test the deployment in a dev enviornment. -## Add component submodule to veda-deploy -Add your component submodule to [.gitmodules](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.gitmodules). Submodules are checked out on the github runner when your component is deployed. - -``` -[submodule "my-project"] - path = my-project - url = git@github.com:NASA-IMPACT/my-project.git -``` - ## Store `.env` configuration in AWS Secrets Manager Custom configurations like RDS instance size as well as AWS environment specific configuration like VPC ID and a Permission Boundary Policy Name should be added to a key-value secret that will be loaded into the github runner environment by your action. This secret should be stored in the target AWS account where the component will be deployed. -> **Note:** +> [!NOTE] > 1. For higher security environments, a permissions boundary policy needs to be identified. > 2. The qualifier of the CDK Toolkit bootstrapped for the target environment must be provided if not using the default toolkit. @@ -121,6 +118,15 @@ STAGE=****** BOOTSTRAP_QUALIFIER=****** ``` +## Add component submodule to veda-deploy +Add your component submodule to [.gitmodules](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.gitmodules). Submodules are checked out on the github runner when your component is deployed. + +``` +[submodule "my-project"] + path = my-project + url = git@github.com:NASA-IMPACT/my-project.git +``` + ## Extend composite dispatched deployment action 1. Add a [dispatch flag in .github/workflows/dispatch.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/dispatch.yml#L66) component you are adding. As in `DEPLOY_MY_COMPONENT: ${{ github.event.inputs.DEPLOY_MY_COMPONENT }}` @@ -131,8 +137,9 @@ BOOTSTRAP_QUALIFIER=****** ## Add new component release version and environment secret name to veda-deploy environment(s) Adding new deployment environments requires admin permissions for this veda-deploy repository. New environments are added by entering project settings and selecting `Environments` from the code and automation menu. The environment naming convention is `-`, i.e. `smce-staging`. As more environments are added this convention will need to be updated. -In the Environment variables for the instance you are dispatching your component to, add a new variable with the github reference to the release you want to deploy. +In the Environment variables for the instance you are dispatching your component to, add a new variable with the github reference to the release you want to deploy. It is best practice to refer to a release tag but a branch name or commit hash can also be used. `MY_COMPONENT_GIT_REF=v1.0` -## Configure domain and custom routes \ No newline at end of file +## Configure domain and custom routes +VEDA platform components include options for custom subdomains and custom root paths. Coordinate how your custom resource should be configured with the team maintaining the target environment you are deploying to. \ No newline at end of file From 429ab4b6b00fed5c327b82c0a88d13920d5c1a5b Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Fri, 31 Jan 2025 09:36:09 -0700 Subject: [PATCH 123/210] formatting --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f95e088..7f81ad5 100644 --- a/README.md +++ b/README.md @@ -78,14 +78,20 @@ VEDA_SM2A_DATA_AIRFLOW_GIT_REF= # Add New Components > [!IMPORTANT] -> This section is intended to expand an existing configured environment, see [How to Deploy](#how-to-deploy) to start from scratch. Please read the full overview before starting, some steps overlap. +> This section is intended to expand an existing configured environment, see [How to Deploy](#how-to-deploy) to start from scratch. Please read the full overview before starting; some steps overlap. ## Overview - [Add deployment action to component github repository](#add-deployment-action-to-component-github-repository) + - [Store `.env` configuration in AWS Secrets Manager](#store-env-configuration-in-aws-secrets-manager) + + - [Add component submodule to veda-deploy](#add-component-submodule-to-veda-deploy) + - [Extend composite dispatched deployment action with an optional component job that uses the component submodule and environment secret](#extend-composite-dispatched-deployment-action) + - [Add new component release version and environment secret name to veda-deploy environment(s)](#add-new-component-release-version-and-environment-secret-name-to-veda-deploy-environments) + - [Configure domain and custom routes](#configure-domain-and-custom-routes) ## Add deployment action to component github repository From 0e43d3be24b30c0b0f21221b44315364784ffeb3 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 31 Jan 2025 11:17:06 -0700 Subject: [PATCH 124/210] changes to variables, how to deploy sections --- README.md | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 7f81ad5..90c86e3 100644 --- a/README.md +++ b/README.md @@ -3,29 +3,20 @@ Deploy full VEDA stack easily. # How to deploy? ## Steps -1. Create an Environment in the repository. See [Requirements](#requirements) on details of creating the environment. +0. To deploy a new component, add the necessary github actions to component repo and add the component as a submodule in veda-deploy. See [Add New Components](#add-new-components). +1. Create a new Github Environment in the repository. See [Requirements](#requirements) on details of creating the environment. 2. Add necessary env vars in the Environment -3. Go to Actions. Select "CI/CD" workflow. Select "Run workflow", choose the environment from step 1. Click "Run workflow." +3. Go to Actions. Select "Dispatch" workflow. Select "Run workflow", choose the environment from step 1. Select the components to dispatch and then "Run workflow." # Requirements ## Environment -Each environment needs a minimum of +Each Github Environment needs a minimum of: ### Secrets `DEPLOYMENT_ROLE_ARN` - oidc role with permissions to deploy ### Variables -`DEPLOYMENT_ENV_SECRET_NAME` - the AWS secrets manager secret name with the required env vars. See AWS Secrets Requirements for what env vars are needed. -`PROJECT_PREFIX` (TBD) -`STAGE` (TBD) - -### Variables (Optional) -Git Ref for each project to use to deploy. Can be branch name, release tag or commit hash. Anything that works with `git checkout`. - -`VEDA_AUTH_GIT_REF` -`VEDA_BACKEND_GIT_REF` -`VEDA_DATA_AIRFLOW_GIT_REF` -`VEDA_FEATURES_API_GIT_REF` +`DEPLOYMENT_ENV_SECRET_NAME` - the AWS secrets manager secret name with the required component env vars. See [AWS Secrets Requirements](#aws-secrets-requirements) for what env vars are needed. #### AWS Secrets Requirements ```bash @@ -45,6 +36,7 @@ VEDA_STAC_PATH_PREFIX=***** VEDA_RASTER_PATH_PREFIX=***** ``` +`SM2A_ENVS_DEPLOYMENT_SECRET_NAME` - the AWS secrets manager secret name with env vars specific to a SM2A deployment. [AWS Secrets Requirements for SM2A](#aws-secrets-requirements-for-sm2a) for what env vars are needed. #### AWS Secrets Requirements for SM2A ```bash @@ -68,9 +60,14 @@ TF_VAR_gh_app_client_secret=****** TF_VAR_gh_team_name=****** TF_VAR_subdomain=****** ``` -##### Github variables -Add these variables to Github environment variables + +Git Ref for each project to use to deploy. Can be branch name, release tag or commit hash. Anything that works with `git checkout`. + ```bash +VEDA_AUTH_GIT_REF= +VEDA_BACKEND_GIT_REF= +VEDA_DATA_AIRFLOW_GIT_REF= +VEDA_FEATURES_API_GIT_REF= DEPLOY_SM2A=true SM2A_ENVS_DEPLOYMENT_SECRET_NAME= VEDA_SM2A_DATA_AIRFLOW_GIT_REF= From aaa0d57cd1f5fb3b54cff277e5f14ea80e5852d1 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 31 Jan 2025 13:36:52 -0700 Subject: [PATCH 125/210] update how to deploy order --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 90c86e3..a5141fa 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@ Deploy full VEDA stack easily. # How to deploy? ## Steps -0. To deploy a new component, add the necessary github actions to component repo and add the component as a submodule in veda-deploy. See [Add New Components](#add-new-components). 1. Create a new Github Environment in the repository. See [Requirements](#requirements) on details of creating the environment. 2. Add necessary env vars in the Environment 3. Go to Actions. Select "Dispatch" workflow. Select "Run workflow", choose the environment from step 1. Select the components to dispatch and then "Run workflow." +4. (Optional) To add a new component in veda-deploy see [Add New Components].(#add-new-components). # Requirements ## Environment @@ -64,15 +64,15 @@ TF_VAR_subdomain=****** Git Ref for each project to use to deploy. Can be branch name, release tag or commit hash. Anything that works with `git checkout`. ```bash -VEDA_AUTH_GIT_REF= -VEDA_BACKEND_GIT_REF= -VEDA_DATA_AIRFLOW_GIT_REF= -VEDA_FEATURES_API_GIT_REF= -DEPLOY_SM2A=true -SM2A_ENVS_DEPLOYMENT_SECRET_NAME= +VEDA_AUTH_GIT_REF= +VEDA_BACKEND_GIT_REF= +VEDA_DATA_AIRFLOW_GIT_REF= +VEDA_FEATURES_API_GIT_REF= VEDA_SM2A_DATA_AIRFLOW_GIT_REF= ``` +`DEPLOY_SM2A=true` - whether to deploy SM2A + # Add New Components > [!IMPORTANT] > This section is intended to expand an existing configured environment, see [How to Deploy](#how-to-deploy) to start from scratch. Please read the full overview before starting; some steps overlap. From 386cf95b2c85f1c9c4ac0b1afdcd376fd3a1f75d Mon Sep 17 00:00:00 2001 From: smohiudd Date: Fri, 31 Jan 2025 13:37:53 -0700 Subject: [PATCH 126/210] fix link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a5141fa..86dd90c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Deploy full VEDA stack easily. 1. Create a new Github Environment in the repository. See [Requirements](#requirements) on details of creating the environment. 2. Add necessary env vars in the Environment 3. Go to Actions. Select "Dispatch" workflow. Select "Run workflow", choose the environment from step 1. Select the components to dispatch and then "Run workflow." -4. (Optional) To add a new component in veda-deploy see [Add New Components].(#add-new-components). +4. (Optional) To add a new component in veda-deploy see [Add New Components](#add-new-components). # Requirements ## Environment From 5c0b5652c01906f2968879f3f6d8e0fe78be6e22 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 3 Feb 2025 10:55:23 -0700 Subject: [PATCH 127/210] Update README.md Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 86dd90c..dcc41a3 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ Add your component submodule to [.gitmodules](https://github.com/NASA-IMPACT/ved ## Add new component release version and environment secret name to veda-deploy environment(s) Adding new deployment environments requires admin permissions for this veda-deploy repository. New environments are added by entering project settings and selecting `Environments` from the code and automation menu. The environment naming convention is `-`, i.e. `smce-staging`. As more environments are added this convention will need to be updated. -In the Environment variables for the instance you are dispatching your component to, add a new variable with the github reference to the release you want to deploy. It is best practice to refer to a release tag but a branch name or commit hash can also be used. +In the Environment variables for the instance you are dispatching your component to, add a new variable with the GitHub reference to the release you want to deploy. It is best practice to refer to a release tag but a branch name or commit hash can also be used. `MY_COMPONENT_GIT_REF=v1.0` From 8688a7d1a489cd30c6b09a719775e4bd23034048 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 3 Feb 2025 10:55:33 -0700 Subject: [PATCH 128/210] Update README.md Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dcc41a3..71e92c3 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ Add your component submodule to [.gitmodules](https://github.com/NASA-IMPACT/ved 1. Add a [dispatch flag in .github/workflows/dispatch.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/dispatch.yml#L66) component you are adding. As in `DEPLOY_MY_COMPONENT: ${{ github.event.inputs.DEPLOY_MY_COMPONENT }}` 2. Update the [dispatch message](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/dispatch.yml#L46) to include your component. Eventually this will get too long and will need some thought but it is currently helpful to filter the actions and identify specific dispatches. 3. Transfer the above dispatch information to the [.github/workflows/deploy.yml workflow_call](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/deploy.yml#L10). The deploy action is called after the environment is set by the dispatch and for production environments is only executed after the dispatch has been approved by a maintainer. -4. Add a new [named job to deploy.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/deploy.yml#L218) that checks the condition the deployment condition for the component and, when true, checks out the deployment action from the component's github repository and passes in any relevant information like the configuration environment secret name. +4. Add a new [named job to deploy.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/deploy.yml#L218) that checks the condition the deployment condition for the component and, when true, checks out the deployment action from the component's GitHub repository and passes in any relevant information like the configuration environment secret name. ## Add new component release version and environment secret name to veda-deploy environment(s) Adding new deployment environments requires admin permissions for this veda-deploy repository. New environments are added by entering project settings and selecting `Environments` from the code and automation menu. The environment naming convention is `-`, i.e. `smce-staging`. As more environments are added this convention will need to be updated. From e419d52eb687e228a06ae42a502ab734a2d6fdb1 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 3 Feb 2025 10:55:42 -0700 Subject: [PATCH 129/210] Update README.md Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 71e92c3..3fdbf2a 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ Add your component submodule to [.gitmodules](https://github.com/NASA-IMPACT/ved ## Extend composite dispatched deployment action -1. Add a [dispatch flag in .github/workflows/dispatch.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/dispatch.yml#L66) component you are adding. As in `DEPLOY_MY_COMPONENT: ${{ github.event.inputs.DEPLOY_MY_COMPONENT }}` +1. Add a [dispatch flag in .github/workflows/dispatch.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/dispatch.yml#L66) for the component you are adding. As in `DEPLOY_MY_COMPONENT: ${{ github.event.inputs.DEPLOY_MY_COMPONENT }}` 2. Update the [dispatch message](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/dispatch.yml#L46) to include your component. Eventually this will get too long and will need some thought but it is currently helpful to filter the actions and identify specific dispatches. 3. Transfer the above dispatch information to the [.github/workflows/deploy.yml workflow_call](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/deploy.yml#L10). The deploy action is called after the environment is set by the dispatch and for production environments is only executed after the dispatch has been approved by a maintainer. 4. Add a new [named job to deploy.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/deploy.yml#L218) that checks the condition the deployment condition for the component and, when true, checks out the deployment action from the component's GitHub repository and passes in any relevant information like the configuration environment secret name. From 034f50a08f4145199d431114065f79aae68907f7 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 3 Feb 2025 10:55:50 -0700 Subject: [PATCH 130/210] Update README.md Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3fdbf2a..889678b 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ BOOTSTRAP_QUALIFIER=****** ``` ## Add component submodule to veda-deploy -Add your component submodule to [.gitmodules](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.gitmodules). Submodules are checked out on the github runner when your component is deployed. +Add your component submodule to [.gitmodules](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.gitmodules). Submodules are checked out on the GitHub runner when your component is deployed. ``` [submodule "my-project"] From 5b1620c0f4c108ba6fffffe5265af65363da5215 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 3 Feb 2025 10:55:58 -0700 Subject: [PATCH 131/210] Update README.md Co-authored-by: Aimee Barciauskas --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 889678b..7d660de 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ Custom configurations like RDS instance size as well as AWS environment specific > 1. For higher security environments, a permissions boundary policy needs to be identified. > 2. The qualifier of the CDK Toolkit bootstrapped for the target environment must be provided if not using the default toolkit. -### Sample environmnt variables +### Sample environment variables ``` VPC_ID=****** PERMISSIONS_BOUNDARY_POLICY_NAME=****** From 50523c1e6687993531d2425de0ddf2547d765908 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 3 Feb 2025 10:56:05 -0700 Subject: [PATCH 132/210] Update README.md Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d660de..c5a4512 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ To keep the components modular, each action should include all necessary steps f - This [CICD workflow in veda-backend](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/workflows/cicd.yml) demonstrates importing the cdk-deploy/action on a merge event to test the deployment in a dev enviornment. ## Store `.env` configuration in AWS Secrets Manager -Custom configurations like RDS instance size as well as AWS environment specific configuration like VPC ID and a Permission Boundary Policy Name should be added to a key-value secret that will be loaded into the github runner environment by your action. This secret should be stored in the target AWS account where the component will be deployed. +Custom configurations like RDS instance size as well as AWS environment specific configuration like VPC ID and a Permission Boundary Policy Name should be added to a key-value secret that will be loaded into the GitHub runner environment by your action. This secret should be stored in the target AWS account where the component will be deployed. > [!NOTE] > 1. For higher security environments, a permissions boundary policy needs to be identified. From 1b24f1ccbdc22c23b8c4e88ba07378623b1385cf Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 3 Feb 2025 10:56:13 -0700 Subject: [PATCH 133/210] Update README.md Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c5a4512..e016e3a 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ To keep the components modular, each action should include all necessary steps f > Most deployments require [custom environment configuration](#store-env-configuration-in-aws-secrets-manager) that can be retrieved from the AWS Secrets Manager for the deployment. See [veda-backend/scripts/get-env.sh](https://github.com/NASA-IMPACT/veda-backend/blob/develop/scripts/get-env.sh) for an example environmennt configuration utility. ### Examples -- Veda-auth [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-auth/blob/main/.github/actions/cdk-deploy/action.yml) provides a simple example of adding configuration from an Aws Secrets Manager secret and running cdk deploy for an imported submodule. +- Veda-auth [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-auth/blob/main/.github/actions/cdk-deploy/action.yml) provides a simple example of adding configuration from an AWS Secrets Manager secret and running `cdk deploy` for an imported submodule. - Veda-backend [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/actions/cdk-deploy/action.yml) contains logic to run tests before deploying components. - This [CICD workflow in veda-backend](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/workflows/cicd.yml) demonstrates importing the cdk-deploy/action on a merge event to test the deployment in a dev enviornment. From fc9e295fa1df7f365214b76598f3403e8ab56c78 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 3 Feb 2025 10:56:21 -0700 Subject: [PATCH 134/210] Update README.md Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e016e3a..616160a 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Create a new `cdk-deploy/action.yml` in the component project's repository. On a To keep the components modular, each action should include all necessary steps for deployment including Python and Node setup steps. While veda-deploy uses the same runner to deploy all components, it should not be assumed that the runner already has all needed installations and environment configuration from other components (unless a dependency is configured for the job using needs: {upstream-job-name}). > [!TIP] -> Most deployments require [custom environment configuration](#store-env-configuration-in-aws-secrets-manager) that can be retrieved from the AWS Secrets Manager for the deployment. See [veda-backend/scripts/get-env.sh](https://github.com/NASA-IMPACT/veda-backend/blob/develop/scripts/get-env.sh) for an example environmennt configuration utility. +> Most deployments require [custom environment configuration](#store-env-configuration-in-aws-secrets-manager) that can be retrieved from the AWS Secrets Manager for the deployment. See [veda-backend/scripts/get-env.sh](https://github.com/NASA-IMPACT/veda-backend/blob/develop/scripts/get-env.sh) for an example environment configuration utility. ### Examples - Veda-auth [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-auth/blob/main/.github/actions/cdk-deploy/action.yml) provides a simple example of adding configuration from an AWS Secrets Manager secret and running `cdk deploy` for an imported submodule. From 1afcd8c3e89258b47f1426b15125cf97440e92e6 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 3 Feb 2025 10:56:29 -0700 Subject: [PATCH 135/210] Update README.md Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 616160a..e57f08b 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ VEDA_SM2A_DATA_AIRFLOW_GIT_REF= ## Add deployment action to component github repository Dispatches from veda-deploy are composed of deployment actions imported from github submodules. The management of all configuration, testing, and deployment concerns is managed within the component's github repository (not in veda-deploy). -Create a new `cdk-deploy/action.yml` in the component project's repository. On a dispatch the configured release version of the project will be checked out and executed on the veda-deploy github runner. +Create a new `cdk-deploy/action.yml` in the component project's repository. On a dispatch, the configured release version of the project will be checked out and executed on the veda-deploy GitHub runner. To keep the components modular, each action should include all necessary steps for deployment including Python and Node setup steps. While veda-deploy uses the same runner to deploy all components, it should not be assumed that the runner already has all needed installations and environment configuration from other components (unless a dependency is configured for the job using needs: {upstream-job-name}). From bd7c71d164dee27e02743032c65da3439750e8df Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 3 Feb 2025 10:56:37 -0700 Subject: [PATCH 136/210] Update README.md Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e57f08b..e1a67cb 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ VEDA_SM2A_DATA_AIRFLOW_GIT_REF= - [Configure domain and custom routes](#configure-domain-and-custom-routes) ## Add deployment action to component github repository -Dispatches from veda-deploy are composed of deployment actions imported from github submodules. The management of all configuration, testing, and deployment concerns is managed within the component's github repository (not in veda-deploy). +Dispatches from veda-deploy are composed of deployment actions imported from GitHub submodules. The management of all configuration, testing, and deployment concerns is managed within the component's GitHub repository (not in veda-deploy). Create a new `cdk-deploy/action.yml` in the component project's repository. On a dispatch, the configured release version of the project will be checked out and executed on the veda-deploy GitHub runner. From 0c3bac4b127c0121f2c1c71f76c1d82408b849da Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 3 Feb 2025 10:57:18 -0700 Subject: [PATCH 137/210] Update README.md Co-authored-by: Aimee Barciauskas --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e1a67cb..0e5347b 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Deploy full VEDA stack easily. # Requirements ## Environment -Each Github Environment needs a minimum of: +Each veda-deploy Github Environment needs a minimum of: ### Secrets `DEPLOYMENT_ROLE_ARN` - oidc role with permissions to deploy From 0f9af00bac04963db89417bccebf76f5a7bfcfd8 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 3 Feb 2025 10:57:51 -0700 Subject: [PATCH 138/210] Update README.md Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0e5347b..7bb7565 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ Add your component submodule to [.gitmodules](https://github.com/NASA-IMPACT/ved 4. Add a new [named job to deploy.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/deploy.yml#L218) that checks the condition the deployment condition for the component and, when true, checks out the deployment action from the component's GitHub repository and passes in any relevant information like the configuration environment secret name. ## Add new component release version and environment secret name to veda-deploy environment(s) -Adding new deployment environments requires admin permissions for this veda-deploy repository. New environments are added by entering project settings and selecting `Environments` from the code and automation menu. The environment naming convention is `-`, i.e. `smce-staging`. As more environments are added this convention will need to be updated. +Adding new deployment environments requires admin permissions for this veda-deploy repository. New environments are added by entering project settings and selecting `Environments` from the code and automation menu. The environment naming convention is `-`, i.e. `smce-staging`. As more environments are added, this convention will need to be updated. In the Environment variables for the instance you are dispatching your component to, add a new variable with the GitHub reference to the release you want to deploy. It is best practice to refer to a release tag but a branch name or commit hash can also be used. From 4ec1b61c1e418b051cc7185474d59b662823d6ab Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Mon, 3 Feb 2025 10:57:58 -0700 Subject: [PATCH 139/210] Update README.md Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7bb7565..f9ca984 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ VEDA_STAC_PATH_PREFIX=***** VEDA_RASTER_PATH_PREFIX=***** ``` -`SM2A_ENVS_DEPLOYMENT_SECRET_NAME` - the AWS secrets manager secret name with env vars specific to a SM2A deployment. [AWS Secrets Requirements for SM2A](#aws-secrets-requirements-for-sm2a) for what env vars are needed. +`SM2A_ENVS_DEPLOYMENT_SECRET_NAME` - the AWS secrets manager secret name with env vars specific to a SM2A deployment. See [AWS Secrets Requirements for SM2A](#aws-secrets-requirements-for-sm2a) for what env vars are needed. #### AWS Secrets Requirements for SM2A ```bash From 10fc3ee91487b23944773a78bea863637c6168e7 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Tue, 4 Feb 2025 11:01:29 -0700 Subject: [PATCH 140/210] emphasize using the core aws secret over custom component based secrets --- README.md | 53 ++++++++++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index f9ca984..084cc07 100644 --- a/README.md +++ b/README.md @@ -9,16 +9,36 @@ Deploy full VEDA stack easily. 4. (Optional) To add a new component in veda-deploy see [Add New Components](#add-new-components). # Requirements -## Environment -Each veda-deploy Github Environment needs a minimum of: -### Secrets +## GitHub Environment +Each veda-deploy Github Environment needs Environment Secrets and Variables configured in the GitHub UI Settings for this veda-deploy project as well as detailed key-value AWS Secrets Manager secret(s) with configuration for the deployment of all components. + +### GitHub Environment Secrets +GitHub Environment secret configured in the GitHub UI settings for this veda-deploy repo. `DEPLOYMENT_ROLE_ARN` - oidc role with permissions to deploy -### Variables -`DEPLOYMENT_ENV_SECRET_NAME` - the AWS secrets manager secret name with the required component env vars. See [AWS Secrets Requirements](#aws-secrets-requirements) for what env vars are needed. +### GitHub Environment Variables +GitHub Environment variables need to be set in the GitHub UI project settings should be configured with the name(s) of AWS Secrets Manager secrets and GitHub references to the versions of each github submodule that should be checked out for each component deployed. + +#### AWS Secrets Manager Name(s) + +`DEPLOYMENT_ENV_SECRET_NAME` - the AWS secrets manager secret name with the required component env vars. See [AWS Secrets Requirements](#aws-secrets-requirements) for what env vars are needed. Note that the individual submodule GitHub repositories should be consulted for the most up to date environment variable names and explanations. + +`SM2A_ENVS_DEPLOYMENT_SECRET_NAME` - the AWS secrets manager secret name with env vars specific to a SM2A deployment. See [AWS Secrets Requirements for SM2A](#aws-secrets-requirements-for-sm2a) for what env vars are needed. + +#### GitHub References +Git Ref for each project to use to deploy. Can be branch name, release tag or commit hash. Anything that works with `git checkout`. Below are some examples of the components that may be configured in a GitHub Environment. + +```bash +VEDA_AUTH_GIT_REF= +VEDA_BACKEND_GIT_REF= +VEDA_FEATURES_API_GIT_REF= +VEDA_SM2A_DATA_AIRFLOW_GIT_REF= +``` #### AWS Secrets Requirements +A single secret is used to store the configuration for all components for a given GitHub Environment. In some cases, an additional secret may be needed if a component does not have uniquely namespaced `.env` parameters and requires custom values--for example, the Self Managed Apache Airflow (SM2A) component requires a separate [SM2A secret](#aws-secrets-requirements-for-sm2a) in the AWS Secrets Manager. + ```bash AWS_ACCOUNT_ID=****** AWS_REGION=****** @@ -36,8 +56,6 @@ VEDA_STAC_PATH_PREFIX=***** VEDA_RASTER_PATH_PREFIX=***** ``` -`SM2A_ENVS_DEPLOYMENT_SECRET_NAME` - the AWS secrets manager secret name with env vars specific to a SM2A deployment. See [AWS Secrets Requirements for SM2A](#aws-secrets-requirements-for-sm2a) for what env vars are needed. - #### AWS Secrets Requirements for SM2A ```bash AIRFLOW_UID=****** @@ -61,17 +79,7 @@ TF_VAR_gh_team_name=****** TF_VAR_subdomain=****** ``` -Git Ref for each project to use to deploy. Can be branch name, release tag or commit hash. Anything that works with `git checkout`. - -```bash -VEDA_AUTH_GIT_REF= -VEDA_BACKEND_GIT_REF= -VEDA_DATA_AIRFLOW_GIT_REF= -VEDA_FEATURES_API_GIT_REF= -VEDA_SM2A_DATA_AIRFLOW_GIT_REF= -``` -`DEPLOY_SM2A=true` - whether to deploy SM2A # Add New Components > [!IMPORTANT] @@ -82,7 +90,6 @@ VEDA_SM2A_DATA_AIRFLOW_GIT_REF= - [Store `.env` configuration in AWS Secrets Manager](#store-env-configuration-in-aws-secrets-manager) - - [Add component submodule to veda-deploy](#add-component-submodule-to-veda-deploy) - [Extend composite dispatched deployment action with an optional component job that uses the component submodule and environment secret](#extend-composite-dispatched-deployment-action) @@ -107,20 +114,12 @@ To keep the components modular, each action should include all necessary steps f - This [CICD workflow in veda-backend](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/workflows/cicd.yml) demonstrates importing the cdk-deploy/action on a merge event to test the deployment in a dev enviornment. ## Store `.env` configuration in AWS Secrets Manager -Custom configurations like RDS instance size as well as AWS environment specific configuration like VPC ID and a Permission Boundary Policy Name should be added to a key-value secret that will be loaded into the GitHub runner environment by your action. This secret should be stored in the target AWS account where the component will be deployed. +AWS environment specific configuration like VPC ID and a Permission Boundary Policy Name are already included in a core key-value secret that can be loaded into the GitHub runner environment by your action. This core secret is set in the GitHub Variable `DEPLOYMENT_ENV_SECRET_NAME` (See [AWS Secrets Requirements](#aws-secrets-requirements) for the core variable names). Additional required configuration variables should be added to this core secret as needed for the new component. If your component requires custom configuration that conflicts with the core secret, a new secret can be configured--see the implementation of a custom secret for [SM2A](#aws-secrets-requirements-for-sm2a). > [!NOTE] > 1. For higher security environments, a permissions boundary policy needs to be identified. > 2. The qualifier of the CDK Toolkit bootstrapped for the target environment must be provided if not using the default toolkit. -### Sample environment variables -``` -VPC_ID=****** -PERMISSIONS_BOUNDARY_POLICY_NAME=****** -STAGE=****** -BOOTSTRAP_QUALIFIER=****** -``` - ## Add component submodule to veda-deploy Add your component submodule to [.gitmodules](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.gitmodules). Submodules are checked out on the GitHub runner when your component is deployed. From e2475a8f8bb53feb3c7397794d463aa43e94bd31 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Thu, 6 Feb 2025 09:51:26 -0500 Subject: [PATCH 141/210] reorder new env content --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 084cc07..d276dec 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Deploy full VEDA stack easily. # Requirements +Adding new deployment environments requires admin permissions for this veda-deploy repository. New environments are added by entering project settings and selecting `Environments` from the code and automation menu. The environment naming convention is `-`, i.e. `smce-staging`. As more environments are added, this convention will need to be updated. + ## GitHub Environment Each veda-deploy Github Environment needs Environment Secrets and Variables configured in the GitHub UI Settings for this veda-deploy project as well as detailed key-value AWS Secrets Manager secret(s) with configuration for the deployment of all components. @@ -136,12 +138,6 @@ Add your component submodule to [.gitmodules](https://github.com/NASA-IMPACT/ved 3. Transfer the above dispatch information to the [.github/workflows/deploy.yml workflow_call](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/deploy.yml#L10). The deploy action is called after the environment is set by the dispatch and for production environments is only executed after the dispatch has been approved by a maintainer. 4. Add a new [named job to deploy.yml](https://github.com/NASA-IMPACT/veda-deploy/blob/dev/.github/workflows/deploy.yml#L218) that checks the condition the deployment condition for the component and, when true, checks out the deployment action from the component's GitHub repository and passes in any relevant information like the configuration environment secret name. -## Add new component release version and environment secret name to veda-deploy environment(s) -Adding new deployment environments requires admin permissions for this veda-deploy repository. New environments are added by entering project settings and selecting `Environments` from the code and automation menu. The environment naming convention is `-`, i.e. `smce-staging`. As more environments are added, this convention will need to be updated. - -In the Environment variables for the instance you are dispatching your component to, add a new variable with the GitHub reference to the release you want to deploy. It is best practice to refer to a release tag but a branch name or commit hash can also be used. - -`MY_COMPONENT_GIT_REF=v1.0` ## Configure domain and custom routes VEDA platform components include options for custom subdomains and custom root paths. Coordinate how your custom resource should be configured with the team maintaining the target environment you are deploying to. \ No newline at end of file From 4cecf0729d09d4a5ee9445d8ae29f54f360246c7 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 6 Feb 2025 07:58:21 -0700 Subject: [PATCH 142/210] Update README.md Co-authored-by: Aimee Barciauskas --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d276dec..44a34a2 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Adding new deployment environments requires admin permissions for this veda-depl Each veda-deploy Github Environment needs Environment Secrets and Variables configured in the GitHub UI Settings for this veda-deploy project as well as detailed key-value AWS Secrets Manager secret(s) with configuration for the deployment of all components. ### GitHub Environment Secrets -GitHub Environment secret configured in the GitHub UI settings for this veda-deploy repo. +GitHub Environment secret(s) configured in the GitHub UI settings for this veda-deploy repo: `DEPLOYMENT_ROLE_ARN` - oidc role with permissions to deploy ### GitHub Environment Variables From df25d42cda0bea1bd3027e702d026496c4510320 Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 6 Feb 2025 07:58:30 -0700 Subject: [PATCH 143/210] Update README.md Co-authored-by: Aimee Barciauskas --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 44a34a2..0b92039 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ GitHub Environment secret(s) configured in the GitHub UI settings for this veda- ### GitHub Environment Variables GitHub Environment variables need to be set in the GitHub UI project settings should be configured with the name(s) of AWS Secrets Manager secrets and GitHub references to the versions of each github submodule that should be checked out for each component deployed. -#### AWS Secrets Manager Name(s) +#### AWS Secrets Manager Secret Name(s) `DEPLOYMENT_ENV_SECRET_NAME` - the AWS secrets manager secret name with the required component env vars. See [AWS Secrets Requirements](#aws-secrets-requirements) for what env vars are needed. Note that the individual submodule GitHub repositories should be consulted for the most up to date environment variable names and explanations. From b1ba8c5616abb470226acc61701554c4e2e49b7e Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 6 Feb 2025 07:58:47 -0700 Subject: [PATCH 144/210] Update README.md Co-authored-by: Aimee Barciauskas --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0b92039..6f07db2 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ TF_VAR_subdomain=****** # Add New Components > [!IMPORTANT] -> This section is intended to expand an existing configured environment, see [How to Deploy](#how-to-deploy) to start from scratch. Please read the full overview before starting; some steps overlap. +> This section is intended to add a new component to an existing configured environment, see [How to Deploy](#how-to-deploy) to start from scratch. Please read the full overview before starting; some steps overlap. ## Overview - [Add deployment action to component github repository](#add-deployment-action-to-component-github-repository) From fdce722cbf5417dbddc48f9c8c7f3b6e48caacfd Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 6 Feb 2025 07:59:15 -0700 Subject: [PATCH 145/210] Update README.md Co-authored-by: Aimee Barciauskas --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6f07db2..d4c4b93 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,7 @@ TF_VAR_subdomain=****** - [Configure domain and custom routes](#configure-domain-and-custom-routes) ## Add deployment action to component github repository -Dispatches from veda-deploy are composed of deployment actions imported from GitHub submodules. The management of all configuration, testing, and deployment concerns is managed within the component's GitHub repository (not in veda-deploy). +Dispatches from veda-deploy are composed of deployment actions imported from the component projects' repository, after it has been installed as [git submodule](https://git-scm.com/book/en/v2/Git-Tools-Submodules). The management of all configuration, testing, and deployment concerns is managed within the component's GitHub repository (not in veda-deploy). Create a new `cdk-deploy/action.yml` in the component project's repository. On a dispatch, the configured release version of the project will be checked out and executed on the veda-deploy GitHub runner. From c5339df0eeda1967e844fabcf9fd1d4bf5a7a90a Mon Sep 17 00:00:00 2001 From: Alexandra Kirk Date: Thu, 6 Feb 2025 08:00:16 -0700 Subject: [PATCH 146/210] Update README.md Co-authored-by: Aimee Barciauskas --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d4c4b93..9a59ff4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,9 @@ GitHub Environment secret(s) configured in the GitHub UI settings for this veda- `DEPLOYMENT_ROLE_ARN` - oidc role with permissions to deploy ### GitHub Environment Variables -GitHub Environment variables need to be set in the GitHub UI project settings should be configured with the name(s) of AWS Secrets Manager secrets and GitHub references to the versions of each github submodule that should be checked out for each component deployed. +GitHub Environment variables need to be set in the GitHub UI project settings. There should be one variable for each AWS Secrets Manager secret name. There should be one variable for each component indicating which GitHub reference to use to deploy that component via checking out that Github reference in the git submodule. + +More instructions on these Github environment variables is provided below. #### AWS Secrets Manager Secret Name(s) From f25f09a5483ee3659e5399f8080851707161000d Mon Sep 17 00:00:00 2001 From: Henry Rodman Date: Fri, 28 Feb 2025 10:48:09 -0600 Subject: [PATCH 147/210] Add titiler-multidim deployment (#83) * Add titiler-multidim as a submodule * add deployment to deploy.yml and dispatch.yml --------- Co-authored-by: Aimee Barciauskas --- .github/workflows/deploy.yml | 45 ++++++++++++++++++++++++++++++++++ .github/workflows/dispatch.yml | 8 +++++- .gitmodules | 5 +++- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index afc2310..87e30d5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -43,6 +43,11 @@ on: required: true default: false description: DEPLOY_MONITORING + DEPLOY_TITILER_MULTIDIM: + type: string + required: true + default: false + description: DEPLOY_TITILER_MULTIDIM concurrency: group: ${{ inputs.environment }} @@ -307,6 +312,45 @@ jobs: dir: "${{ env.DIRECTORY }}" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + deploy-titiler-multidim: + name: Deploy titiler-multidim 🌎 + runs-on: ubuntu-latest + env: + DIRECTORY: titiler-multidim + ENVIRONMENT: ${{ inputs.environment }} + environment: ${{ inputs.environment }} + if: ${{ inputs.DEPLOY_TITILER_MULTIDIM == 'true' }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: "true" + submodules: "false" + + - name: Checkout titiler-multidim submodule + uses: actions/checkout@v4 + with: + repository: "developmentseed/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.TITILER_MULTIDIM_GIT_REF || 'main'}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-titiler-multidim-deployment" + aws-region: "us-west-2" + + - name: Run titiler-multidim deployment + id: deploy_titiler_multidim + uses: "./titiler-multidim/.github/actions/cdk-deploy" + with: + dir: "${{ env.DIRECTORY }}/infrastructure/aws" + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + + deploy-veda-routes: name: deploy VEDA routes πŸ”€ runs-on: ubuntu-latest @@ -478,3 +522,4 @@ jobs: name: playwright-report path: playwright-report/ retention-days: 30 + diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 66eeea6..6b6a070 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -42,8 +42,13 @@ on: required: true default: false description: DEPLOY_MONITORING + DEPLOY_TITILER_MULTIDIM: + type: boolean + required: true + default: false + description: DEPLOY_TITILER_MULTIDIM -run-name: Dispatch to ${{ inputs.environment }} AUTH=${{ inputs.DEPLOY_AUTH }} BACKEND=${{ inputs.DEPLOY_BACKEND }} FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} β›Ÿ +run-name: Dispatch to ${{ inputs.environment }} AUTH=${{ inputs.DEPLOY_AUTH }} BACKEND=${{ inputs.DEPLOY_BACKEND }} FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} TITILER_MULTIDIM=${{ inputs.DEPLOY_TITILER_MULTIDIM }} β›Ÿ jobs: check-environment: @@ -75,4 +80,5 @@ jobs: DEPLOY_ROUTES: ${{ github.event.inputs.DEPLOY_ROUTES }} DEPLOY_SM2A: ${{ github.event.inputs.DEPLOY_SM2A }} DEPLOY_MONITORING: ${{ github.event.inputs.DEPLOY_MONITORING }} + DEPLOY_TITILER_MULTIDIM: ${{ github.event.inputs.DEPLOY_TITILER_MULTIDIM }} secrets: inherit diff --git a/.gitmodules b/.gitmodules index bd4080f..cb98e23 100644 --- a/.gitmodules +++ b/.gitmodules @@ -15,4 +15,7 @@ url = git@github.com:NASA-IMPACT/veda-features-api-cdk.git [submodule "veda-routes"] path = veda-routes - url = git@github.com:NASA-IMPACT/veda-routes.git + url = git@github.com:NASA-IMPACT/veda-routes.git +[submodule "titiler-multidim"] + path = titiler-multidim + url = git@github.com:developmentseed/titiler-multidim.git From a8b902da2e85a98ca96d57f250860d0d2adf538a Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Mon, 17 Mar 2025 15:18:54 -0700 Subject: [PATCH 148/210] fix: update workflows to use full commit hash --- .github/workflows/deploy.yml | 64 ++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 87e30d5..5fedc79 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -68,20 +68,20 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: lfs: "true" submodules: "false" - name: Checkout veda-auth submodule - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" @@ -116,20 +116,20 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: lfs: "true" submodules: "false" - name: Checkout veda-backend submodule - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} ref: ${{ vars.VEDA_BACKEND_GIT_REF || 'main'}} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-backend-deployment" @@ -182,20 +182,20 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: lfs: "true" submodules: "false" - name: Checkout veda-data-airflow submodule - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} ref: ${{ vars.VEDA_SM2A_DATA_AIRFLOW_GIT_REF || 'main'}} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-airflow-sm2a-deployment" @@ -231,20 +231,20 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: lfs: "true" submodules: "false" - name: Checkout veda-features-api submodule - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} ref: ${{ vars.VEDA_FEATURES_API_GIT_REF || 'main'}} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-features-deployment" @@ -282,14 +282,14 @@ jobs: steps: - name: Checkout if: ${{ env.GH_PAT_CHECK != '' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: lfs: "true" submodules: "false" - name: Checkout veda-monitoring "submodule" if: ${{ env.GH_PAT_CHECK != '' }} - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} @@ -298,7 +298,7 @@ jobs: - name: Configure AWS Credentials if: ${{ env.GH_PAT_CHECK != '' }} - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-monitoring-deployment" @@ -323,20 +323,20 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: lfs: "true" submodules: "false" - name: Checkout titiler-multidim submodule - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "developmentseed/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} ref: ${{ vars.TITILER_MULTIDIM_GIT_REF || 'main'}} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-titiler-multidim-deployment" @@ -364,13 +364,13 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: lfs: "true" submodules: "false" - name: Checkout veda-routes "submodule" - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: "NASA-IMPACT/${{ env.DIRECTORY }}" path: ${{ env.DIRECTORY }} @@ -378,19 +378,19 @@ jobs: token: ${{ secrets.GH_PAT }} - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-routes-deployment" aws-region: "${{ env.AWS_REGION }}" - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa #v4.8.0 with: python-version: "3.10" - name: Setup python cache - uses: actions/cache@v4 + uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf #v4.2.2 with: path: ${{ env.pythonLocation }} key: ${{ env.pythonLocation }}-${{ hashFiles('${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt') }} @@ -411,7 +411,7 @@ jobs: --raster_api_url=${{ needs.deploy-veda-backend.outputs.raster_api_url }} \ --ingest_api_url=${{ needs.deploy-veda-backend.outputs.ingest_api_url }} \ --stac_api_url=${{ needs.deploy-veda-backend.outputs.stac_api_url }} \ - --stac_browser_bucket_name=${{ needs.deploy-veda-backend.outputs.stac_browser_bucket_name }} + --stac_browser_bucket_name=${{ needs.deploy-veda-backend.outputs.stac_browser_bucket_name }} - name: Run deployment uses: "./veda-routes/.github/actions/cdk-deploy" @@ -430,10 +430,10 @@ jobs: AWS_DEFAULT_REGION: us-west-2 environment: ${{ inputs.environment }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 #v2.7.0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa #v4.8.0 with: python-version: "3.10" cache: "pip" @@ -441,7 +441,7 @@ jobs: ${{ github.workspace }}/${{ env.DIRECTORY }}/requirements.txt - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-integration-test" @@ -467,13 +467,13 @@ jobs: needs: [ deploy-veda-backend ] steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: NASA-IMPACT/veda-config ref: add-playwright - name: Use Node.js 16 - uses: actions/setup-node@v3 + uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 #v3.8.2 with: node-version: 16 @@ -481,7 +481,7 @@ jobs: run: ./.veda/setup - name: Checkout generate_env script - uses: actions/checkout@v4 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: sparse-checkout: | /scripts/generate_env_file.py @@ -496,7 +496,7 @@ jobs: - name: Load .env file id: dotenv - uses: falti/dotenv-action@v1.1 + uses: falti/dotenv-action@a33be0b8cf6a6e6f1b82cc9f3782061ab1022be5 #v1.1.4 - name: Create env.local file env: @@ -516,7 +516,7 @@ jobs: - name: Playwright tests run: yarn test:e2e - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@c24449f33cd45d4826c6702db7e49f7cdb9b551d #v3.2.1-node20 if: always() with: name: playwright-report From f1cf9fd5599ad46e9847ee79b1f174ed26c9c582 Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Mon, 17 Mar 2025 15:21:22 -0700 Subject: [PATCH 149/210] fix: add uses --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5fedc79..739de83 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -81,7 +81,7 @@ jobs: ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} - name: Configure AWS Credentials - aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 with: role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" From 10bd8cdfa856b71aa02413ce21b6c71b25a40f8f Mon Sep 17 00:00:00 2001 From: Abdelhak Marouane Date: Fri, 21 Mar 2025 16:34:51 -0500 Subject: [PATCH 150/210] Add submodule of titiler for tracking --- .gitmodules | 2 +- titiler-multidim | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 160000 titiler-multidim diff --git a/.gitmodules b/.gitmodules index cb98e23..0b8a17e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -18,4 +18,4 @@ url = git@github.com:NASA-IMPACT/veda-routes.git [submodule "titiler-multidim"] path = titiler-multidim - url = git@github.com:developmentseed/titiler-multidim.git + url = https://github.com/developmentseed/titiler-multidim.git diff --git a/titiler-multidim b/titiler-multidim new file mode 160000 index 0000000..0e0181b --- /dev/null +++ b/titiler-multidim @@ -0,0 +1 @@ +Subproject commit 0e0181b13f985a9fcd5f4c6c0681342ae52d5369 From 32f0ca7c9c32b06c0834800bdc5275706d159996 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Fri, 4 Apr 2025 13:30:07 -0500 Subject: [PATCH 151/210] add s3 submodule --- .gitmodules | 3 +++ s3-disaster-recovery | 1 + 2 files changed, 4 insertions(+) create mode 160000 s3-disaster-recovery diff --git a/.gitmodules b/.gitmodules index 0b8a17e..bd1072e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,6 @@ [submodule "titiler-multidim"] path = titiler-multidim url = https://github.com/developmentseed/titiler-multidim.git +[submodule "s3-disaster-recovery"] + path = s3-disaster-recovery + url = https://github.com/NASA-IMPACT/s3-disaster-recovery.git diff --git a/s3-disaster-recovery b/s3-disaster-recovery new file mode 160000 index 0000000..958c137 --- /dev/null +++ b/s3-disaster-recovery @@ -0,0 +1 @@ +Subproject commit 958c1376dcf604e2c0d420a920e46f9a89918515 From 9804eb906ab85758966bfb9cc09406cebf870737 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Fri, 4 Apr 2025 13:31:55 -0500 Subject: [PATCH 152/210] add s3 disaster dispatch file --- .github/workflows/dispatch.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 6b6a070..09fb7fb 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -46,7 +46,12 @@ on: type: boolean required: true default: false - description: DEPLOY_TITILER_MULTIDIM + description: DEPLOY_TITILER_MULTIDIM + DEPLOY_S3_DISASTER_RECOVERY: + type: boolean + required: true + default: false + description: DEPLOY_S3_DISASTER_RECOVERY run-name: Dispatch to ${{ inputs.environment }} AUTH=${{ inputs.DEPLOY_AUTH }} BACKEND=${{ inputs.DEPLOY_BACKEND }} FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} TITILER_MULTIDIM=${{ inputs.DEPLOY_TITILER_MULTIDIM }} β›Ÿ @@ -81,4 +86,5 @@ jobs: DEPLOY_SM2A: ${{ github.event.inputs.DEPLOY_SM2A }} DEPLOY_MONITORING: ${{ github.event.inputs.DEPLOY_MONITORING }} DEPLOY_TITILER_MULTIDIM: ${{ github.event.inputs.DEPLOY_TITILER_MULTIDIM }} + DEPLOY_S3_DISASTER_RECOVERY: ${{ github.event.inputs.DEPLOY_S3_DISASTER_RECOVERY }} secrets: inherit From aca0e81a21addf434f539c85dd03ca13f9c860a9 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Fri, 4 Apr 2025 13:50:10 -0500 Subject: [PATCH 153/210] add for s3 disaster --- .github/workflows/deploy.yml | 54 ++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 739de83..4dc1402 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -48,6 +48,11 @@ on: required: true default: false description: DEPLOY_TITILER_MULTIDIM + DEPLOY_S3_DISASTER_RECOVERY: + type: string + required: true + default: false + description: DEPLOY_S3_DISASTER_RECOVERY concurrency: group: ${{ inputs.environment }} @@ -350,6 +355,55 @@ jobs: env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + deploy-s3-disaster-recovery: + name: Deploy s3-disaster-recovery + runs-on: ubuntu-latest + env: + DIRECTORY: s3-disaster-recovery + ENVIRONMENT: ${{ inputs.environment }} + environment: ${{ inputs.environment }} + if: ${{ inputs.DEPLOY_S3_DISASTER_RECOVERY == 'true' }} + + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + lfs: "true" + submodules: "false" + + - name: Checkout s3-disaster-recovery submodule + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.S3_DISASTER_RECOVERY_GIT_REF || 'main'}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-s3-disaster-recovery-deployment" + aws-region: "us-west-2" + + - name: Run s3-disaster-recovery deployment + id: deploy_s3-disaster-recovery + uses: "./.github/workflows/s3-disaster-recovery" + with: + dir: "${{ env.DIRECTORY }}" + script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + source-bucket: ${{ vars.SOURCE_BUCKET_NAME }} + destination-bucket: ${{ vars.DESTINATION_BUCKET_NAME }} + allow-batch-replication: ${{ vars.ALLOW_BATCH_REPLICATION }} + aws-region: ${{ vars.AWS_REGION }} + secrets: + deployment-account: ${{ secrets.DEPLOYMENT_ACCOUNT }} + deployment-role: ${{ secrets.DEPLOYMENT_ROLE_NAME }} + permissions-boundary: ${{ secrets.PERMISSIONS_BOUNDARY_ARN }} + + # with: + # dir: "${{ env.DIRECTORY }}/infrastructure/aws" + # env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + # script_path: "${{ github.workspace }}/scripts/generate_env_file.py" deploy-veda-routes: name: deploy VEDA routes πŸ”€ From 08ba76530f1e530e64487cb672725c905b3f92b6 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Fri, 4 Apr 2025 16:27:57 -0500 Subject: [PATCH 154/210] pass env to s3 disaster actions --- .github/workflows/deploy.yml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4dc1402..06555c2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -389,21 +389,14 @@ jobs: id: deploy_s3-disaster-recovery uses: "./.github/workflows/s3-disaster-recovery" with: - dir: "${{ env.DIRECTORY }}" - script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + env-file: ".env" + aws-region: ${{ vars.AWS_REGION }} + allow-batch-replication: ${{ vars.ALLOW_BATCH_REPLICATION }} source-bucket: ${{ vars.SOURCE_BUCKET_NAME }} destination-bucket: ${{ vars.DESTINATION_BUCKET_NAME }} - allow-batch-replication: ${{ vars.ALLOW_BATCH_REPLICATION }} - aws-region: ${{ vars.AWS_REGION }} - secrets: - deployment-account: ${{ secrets.DEPLOYMENT_ACCOUNT }} - deployment-role: ${{ secrets.DEPLOYMENT_ROLE_NAME }} - permissions-boundary: ${{ secrets.PERMISSIONS_BOUNDARY_ARN }} - - # with: - # dir: "${{ env.DIRECTORY }}/infrastructure/aws" - # env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - # script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + dir: "${{ env.DIRECTORY }}" + script_path: "${{ github.workspace }}/scripts/generate_env_file.py" deploy-veda-routes: name: deploy VEDA routes πŸ”€ From 2f5890b6dceefeb3ba4cb58d40da951163b0e3cb Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 11:38:52 -0500 Subject: [PATCH 155/210] fixa action name --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 06555c2..7341033 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -387,7 +387,7 @@ jobs: - name: Run s3-disaster-recovery deployment id: deploy_s3-disaster-recovery - uses: "./.github/workflows/s3-disaster-recovery" + uses: "./.github/workflows/action" with: env-file: ".env" aws-region: ${{ vars.AWS_REGION }} From 739b580335091f8d52f109ae25a6e32e336c0830 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 11:44:39 -0500 Subject: [PATCH 156/210] fix action name --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 7341033..ce5c85d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -387,7 +387,7 @@ jobs: - name: Run s3-disaster-recovery deployment id: deploy_s3-disaster-recovery - uses: "./.github/workflows/action" + uses: "./.github/workflows/action.yml" with: env-file: ".env" aws-region: ${{ vars.AWS_REGION }} From c5ca8e9dc9c706f066b60ffd05fb3c811a654739 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 11:47:01 -0500 Subject: [PATCH 157/210] fix action name --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index ce5c85d..4ef4d5f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -387,7 +387,7 @@ jobs: - name: Run s3-disaster-recovery deployment id: deploy_s3-disaster-recovery - uses: "./.github/workflows/action.yml" + uses: "./s3-diasater-recovery/.github/workflows/action.yml" with: env-file: ".env" aws-region: ${{ vars.AWS_REGION }} From c18c9305b70e76f1187c9118a3d753f57d04b948 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 12:30:16 -0500 Subject: [PATCH 158/210] update repo --- .github/workflows/deploy.yml | 2 +- s3-disaster-recovery | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4ef4d5f..a10c147 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -387,7 +387,7 @@ jobs: - name: Run s3-disaster-recovery deployment id: deploy_s3-disaster-recovery - uses: "./s3-diasater-recovery/.github/workflows/action.yml" + uses: "./s3-diasater-recovery/.github/workflows/s3-disaster-recovery" with: env-file: ".env" aws-region: ${{ vars.AWS_REGION }} diff --git a/s3-disaster-recovery b/s3-disaster-recovery index 958c137..1061586 160000 --- a/s3-disaster-recovery +++ b/s3-disaster-recovery @@ -1 +1 @@ -Subproject commit 958c1376dcf604e2c0d420a920e46f9a89918515 +Subproject commit 1061586f1d9e44e7027ce6543b82970c70f95778 From f1ba296c284c8651525ea2d655357dcddc61a4dc Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 12:38:51 -0500 Subject: [PATCH 159/210] fix typo --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a10c147..f04dc00 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -387,7 +387,7 @@ jobs: - name: Run s3-disaster-recovery deployment id: deploy_s3-disaster-recovery - uses: "./s3-diasater-recovery/.github/workflows/s3-disaster-recovery" + uses: "./s3-disaster-recovery/.github/workflows/s3-disaster-recovery" with: env-file: ".env" aws-region: ${{ vars.AWS_REGION }} From 5f2e16e7150ea06359c1b426d6c4850231f8abd5 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 12:41:19 -0500 Subject: [PATCH 160/210] action name --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f04dc00..f4b75ad 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -387,7 +387,7 @@ jobs: - name: Run s3-disaster-recovery deployment id: deploy_s3-disaster-recovery - uses: "./s3-disaster-recovery/.github/workflows/s3-disaster-recovery" + uses: "./s3-disaster-recovery/.github/workflows" with: env-file: ".env" aws-region: ${{ vars.AWS_REGION }} From 2613bd1363d3d311e5dbbe17cc7e0c68a19dcbdc Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 12:45:22 -0500 Subject: [PATCH 161/210] ... --- s3-disaster-recovery | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3-disaster-recovery b/s3-disaster-recovery index 1061586..efca4f9 160000 --- a/s3-disaster-recovery +++ b/s3-disaster-recovery @@ -1 +1 @@ -Subproject commit 1061586f1d9e44e7027ce6543b82970c70f95778 +Subproject commit efca4f9a919082b18f9e5d672489968eb5cce774 From 71a56177915f20410e79602ed5130ca6ca62ea05 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 13:04:20 -0500 Subject: [PATCH 162/210] update --- s3-disaster-recovery | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3-disaster-recovery b/s3-disaster-recovery index efca4f9..74ce7df 160000 --- a/s3-disaster-recovery +++ b/s3-disaster-recovery @@ -1 +1 @@ -Subproject commit efca4f9a919082b18f9e5d672489968eb5cce774 +Subproject commit 74ce7dfb6e38b6cb79d2942fa87a45dfcaed50c8 From 1903627eaad76c795e3c053898a87948e1a38ba5 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 13:10:52 -0500 Subject: [PATCH 163/210] ... --- s3-disaster-recovery | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3-disaster-recovery b/s3-disaster-recovery index 74ce7df..4297e94 160000 --- a/s3-disaster-recovery +++ b/s3-disaster-recovery @@ -1 +1 @@ -Subproject commit 74ce7dfb6e38b6cb79d2942fa87a45dfcaed50c8 +Subproject commit 4297e94b5501e69b87605c052d06e34fde065601 From 69760b8728c922b8aaf94bf9d0800700c9beda49 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 13:16:49 -0500 Subject: [PATCH 164/210] fix vars --- .github/workflows/deploy.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f4b75ad..46849cf 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -389,11 +389,11 @@ jobs: id: deploy_s3-disaster-recovery uses: "./s3-disaster-recovery/.github/workflows" with: - env-file: ".env" - aws-region: ${{ vars.AWS_REGION }} - allow-batch-replication: ${{ vars.ALLOW_BATCH_REPLICATION }} - source-bucket: ${{ vars.SOURCE_BUCKET_NAME }} - destination-bucket: ${{ vars.DESTINATION_BUCKET_NAME }} + env_file: ".env" + aws_region: ${{ vars.AWS_REGION }} + allow_batch_replication: ${{ vars.ALLOW_BATCH_REPLICATION }} + source_bucket: ${{ vars.SOURCE_BUCKET_NAME }} + destination_bucket: ${{ vars.DESTINATION_BUCKET_NAME }} env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} dir: "${{ env.DIRECTORY }}" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" From 3b0297c087920d8a75130a12bd7f3c6da5279a72 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 13:20:51 -0500 Subject: [PATCH 165/210] new run --- s3-disaster-recovery | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3-disaster-recovery b/s3-disaster-recovery index 4297e94..4d50b89 160000 --- a/s3-disaster-recovery +++ b/s3-disaster-recovery @@ -1 +1 @@ -Subproject commit 4297e94b5501e69b87605c052d06e34fde065601 +Subproject commit 4d50b8970f33c071a88eb3b7e1ad92507a4ce6c5 From 00bd16073a23dab17ea937546a1d3c41ed6ab82b Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 13:24:12 -0500 Subject: [PATCH 166/210] ... --- s3-disaster-recovery | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3-disaster-recovery b/s3-disaster-recovery index 4d50b89..43a4d46 160000 --- a/s3-disaster-recovery +++ b/s3-disaster-recovery @@ -1 +1 @@ -Subproject commit 4d50b8970f33c071a88eb3b7e1ad92507a4ce6c5 +Subproject commit 43a4d4650bc1227a3d95be01f9fd886bd0597653 From c6a52779ea4ed74babd1ca70fb165194a3549881 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 15:49:56 -0500 Subject: [PATCH 167/210] ... --- s3-disaster-recovery | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3-disaster-recovery b/s3-disaster-recovery index 43a4d46..de8ec58 160000 --- a/s3-disaster-recovery +++ b/s3-disaster-recovery @@ -1 +1 @@ -Subproject commit 43a4d4650bc1227a3d95be01f9fd886bd0597653 +Subproject commit de8ec58a95d89821f7587176e6573104fe454a77 From 960df988cb3219c40ce65df23a2b0e6b4c823773 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 7 Apr 2025 16:26:57 -0500 Subject: [PATCH 168/210] ... --- s3-disaster-recovery | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3-disaster-recovery b/s3-disaster-recovery index de8ec58..539b822 160000 --- a/s3-disaster-recovery +++ b/s3-disaster-recovery @@ -1 +1 @@ -Subproject commit de8ec58a95d89821f7587176e6573104fe454a77 +Subproject commit 539b8226b1ee3eb3cda70500d683c1cdb76aea56 From 921158bda33b2747b2663653a21ca35b7d3199be Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Mon, 14 Apr 2025 11:44:00 -0500 Subject: [PATCH 169/210] pull latest --- s3-disaster-recovery | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s3-disaster-recovery b/s3-disaster-recovery index 539b822..1488ec0 160000 --- a/s3-disaster-recovery +++ b/s3-disaster-recovery @@ -1 +1 @@ -Subproject commit 539b8226b1ee3eb3cda70500d683c1cdb76aea56 +Subproject commit 1488ec04ab924968d6c677a3d73fc52703c5abc6 From fa141860e13b6292a92da27ba4271f6c6a3ad760 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Tue, 6 May 2025 15:45:11 -0600 Subject: [PATCH 170/210] remove auth dependency from veda-backend --- .github/workflows/deploy.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 46849cf..c2fed35 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -115,7 +115,6 @@ jobs: env: DIRECTORY: veda-backend ENVIRONMENT: ${{ inputs.environment }} - needs: [deploy-veda-auth] if: ${{ inputs.DEPLOY_BACKEND == 'true' }} environment: ${{ inputs.environment }} From 6a04d16961416227eace432b9bdeb5dc72217b40 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Tue, 6 May 2025 18:21:11 -0600 Subject: [PATCH 171/210] remove auth stack name from veda-backend --- .github/workflows/deploy.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c2fed35..4ab7962 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -144,7 +144,6 @@ jobs: id: deploy_backend_stack with: env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} - auth_stack_name: "${{ needs.deploy-veda-auth.outputs.auth_stack_name }}" dir: "${{ env.DIRECTORY }}" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" From de1ea591115e9fbd1d13c8d1011f92f6399f846f Mon Sep 17 00:00:00 2001 From: smohiudd Date: Wed, 7 May 2025 10:13:48 -0600 Subject: [PATCH 172/210] make stack name arg optional --- .github/workflows/deploy.yml | 2 +- .github/workflows/dispatch.yml | 4 ++-- scripts/generate_env_file.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4ab7962..ab25880 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -16,7 +16,7 @@ on: DEPLOY_AUTH: type: string required: true - default: true + default: false description: DEPLOY_AUTH DEPLOY_BACKEND: type: string diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 09fb7fb..b2cb0f9 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -15,7 +15,7 @@ on: DEPLOY_AUTH: type: boolean required: true - default: true + default: false description: DEPLOY_AUTH DEPLOY_BACKEND: type: boolean @@ -68,7 +68,7 @@ jobs: # for the approval cannot exceed 60 minutes or the job will fail due to bad credentials with: secret: ${{ secrets.GITHUB_TOKEN }} - approvers: amarouane-ABDELHAK,slesaad,anayeaye,smohiudd,ciaransweet,botanical,ividito,stephenkilbourn + approvers: amarouane-ABDELHAK,slesaad,anayeaye,smohiudd,botanical,ividito,stephenkilbourn minimum-approvals: 1 issue-title: "Deploying to ${{ github.event.inputs.environment }}" issue-body: "Please approve or deny the deployment" diff --git a/scripts/generate_env_file.py b/scripts/generate_env_file.py index 7639c9e..894d815 100644 --- a/scripts/generate_env_file.py +++ b/scripts/generate_env_file.py @@ -52,6 +52,7 @@ def generate_env_file(secret_id, stack_names=None, out_file=".env"): "--stack-names", dest="stack_names", help="Cloudformation Stack names (comma separated)", + required=False, default=None, ) parser.add_argument( From 76a0d41db77c752d2ab8f100145548688b5ea121 Mon Sep 17 00:00:00 2001 From: smohiudd Date: Thu, 15 May 2025 09:25:29 -0600 Subject: [PATCH 173/210] optional stack names arg --- scripts/generate_env_file.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/generate_env_file.py b/scripts/generate_env_file.py index 894d815..4fb86aa 100644 --- a/scripts/generate_env_file.py +++ b/scripts/generate_env_file.py @@ -51,9 +51,11 @@ def generate_env_file(secret_id, stack_names=None, out_file=".env"): parser.add_argument( "--stack-names", dest="stack_names", - help="Cloudformation Stack names (comma separated)", + help="Cloudformation Stack names (comma separated). If the flag is used without a value, or if the flag is omitted entirely, it defaults to None.", required=False, default=None, + nargs='?', + const=None, ) parser.add_argument( "--env-file", From a90da880ce0765309812fd9fd613a04d3fdc4862 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 20 May 2025 19:04:54 -0700 Subject: [PATCH 174/210] Working on deployment for titiler-cmr --- .github/workflows/deploy.yml | 44 ++++++++++++++++++++++++++++++++++ .github/workflows/dispatch.yml | 8 ++++++- .gitmodules | 3 +++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 46849cf..96bf296 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,6 +53,11 @@ on: required: true default: false description: DEPLOY_S3_DISASTER_RECOVERY + DEPLOY_TITILER_CNR: + type: string + required: true + default: false + description: DEPLOY_TITILER_CNR concurrency: group: ${{ inputs.environment }} @@ -398,6 +403,45 @@ jobs: dir: "${{ env.DIRECTORY }}" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + deploy-titiler-cmr: + name: Deploy titiler-cmr 🌎 + runs-on: ubuntu-latest + env: + DIRECTORY: titiler-cmr + ENVIRONMENT: ${{ inputs.environment }} + environment: ${{ inputs.environment }} + if: ${{ inputs.DEPLOY_TITILER_CMR == 'true' }} + + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + # curious why this is needed + lfs: "true" + submodules: "false" + + - name: Checkout titiler-cmr submodule + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: "developmentseed/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.TITILER_CMR_GIT_REF || 'develop'}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-titiler-cmr-deployment" + aws-region: "us-west-2" + + - name: Run titiler-cmr deployment + id: deploy_titiler_cmr + uses: "./titiler-cmr/.github/actions/cdk-deploy" + with: + dir: "${{ env.DIRECTORY }}/infrastructure/aws" + env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} + script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + deploy-veda-routes: name: deploy VEDA routes πŸ”€ runs-on: ubuntu-latest diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 09fb7fb..279d797 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -51,7 +51,12 @@ on: type: boolean required: true default: false - description: DEPLOY_S3_DISASTER_RECOVERY + description: DEPLOY_S3_DISASTER_RECOVERY + DEPLOY_TITILER_CNR: + type: boolean + required: true + default: false + description: DEPLOY_TITILER_CNR run-name: Dispatch to ${{ inputs.environment }} AUTH=${{ inputs.DEPLOY_AUTH }} BACKEND=${{ inputs.DEPLOY_BACKEND }} FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} TITILER_MULTIDIM=${{ inputs.DEPLOY_TITILER_MULTIDIM }} β›Ÿ @@ -87,4 +92,5 @@ jobs: DEPLOY_MONITORING: ${{ github.event.inputs.DEPLOY_MONITORING }} DEPLOY_TITILER_MULTIDIM: ${{ github.event.inputs.DEPLOY_TITILER_MULTIDIM }} DEPLOY_S3_DISASTER_RECOVERY: ${{ github.event.inputs.DEPLOY_S3_DISASTER_RECOVERY }} + DEPLOY_TITILER_CNR: ${{ github.event.inputs.DEPLOY_TITILER_CNR }} secrets: inherit diff --git a/.gitmodules b/.gitmodules index bd1072e..6fd90a8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,3 +22,6 @@ [submodule "s3-disaster-recovery"] path = s3-disaster-recovery url = https://github.com/NASA-IMPACT/s3-disaster-recovery.git +[submodule "titiler-cmr"] + path = titiler-cmr + url = https://github.com/developmentseed/titiler-cmr.git From 2be4bdf0a9b23acd66a2883826b768a6f0674d10 Mon Sep 17 00:00:00 2001 From: ividito Date: Tue, 13 May 2025 13:36:44 -0300 Subject: [PATCH 175/210] flatten airflow repo --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 46849cf..5731fb7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -212,7 +212,7 @@ jobs: with: env-file: ".env" env_aws_secret_name: ${{ vars.SM2A_ENVS_DEPLOYMENT_SECRET_NAME }} - dir: "${{ env.DIRECTORY }}/sm2a" + dir: "${{ env.DIRECTORY }}" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" - name: Get SM2A Workflows API Endpoint From 92aae3b4f864503361064c829b89db3b0c839ec9 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 21 May 2025 14:33:28 -0700 Subject: [PATCH 176/210] Update emoji for titiler-cmr --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 96bf296..2c6552d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -404,7 +404,7 @@ jobs: script_path: "${{ github.workspace }}/scripts/generate_env_file.py" deploy-titiler-cmr: - name: Deploy titiler-cmr 🌎 + name: Deploy titiler-cmr πŸ“š runs-on: ubuntu-latest env: DIRECTORY: titiler-cmr From 5f53d2d32f662e9bda7ebb2f59c4d5d8fce69a0f Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 21 May 2025 14:37:43 -0700 Subject: [PATCH 177/210] Fix CNR->CMR --- .github/workflows/deploy.yml | 4 ++-- .github/workflows/dispatch.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2c6552d..5eca565 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -53,11 +53,11 @@ on: required: true default: false description: DEPLOY_S3_DISASTER_RECOVERY - DEPLOY_TITILER_CNR: + DEPLOY_TITILER_CMR: type: string required: true default: false - description: DEPLOY_TITILER_CNR + description: DEPLOY_TITILER_CMR concurrency: group: ${{ inputs.environment }} diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 279d797..29a5323 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -52,11 +52,11 @@ on: required: true default: false description: DEPLOY_S3_DISASTER_RECOVERY - DEPLOY_TITILER_CNR: + DEPLOY_TITILER_CMR: type: boolean required: true default: false - description: DEPLOY_TITILER_CNR + description: DEPLOY_TITILER_CMR run-name: Dispatch to ${{ inputs.environment }} AUTH=${{ inputs.DEPLOY_AUTH }} BACKEND=${{ inputs.DEPLOY_BACKEND }} FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} TITILER_MULTIDIM=${{ inputs.DEPLOY_TITILER_MULTIDIM }} β›Ÿ @@ -92,5 +92,5 @@ jobs: DEPLOY_MONITORING: ${{ github.event.inputs.DEPLOY_MONITORING }} DEPLOY_TITILER_MULTIDIM: ${{ github.event.inputs.DEPLOY_TITILER_MULTIDIM }} DEPLOY_S3_DISASTER_RECOVERY: ${{ github.event.inputs.DEPLOY_S3_DISASTER_RECOVERY }} - DEPLOY_TITILER_CNR: ${{ github.event.inputs.DEPLOY_TITILER_CNR }} + DEPLOY_TITILER_CMR: ${{ github.event.inputs.DEPLOY_TITILER_CMR }} secrets: inherit From 30b67bc64507fda0e02fe379a0afcd8c1ae2c858 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Thu, 22 May 2025 09:41:52 -0700 Subject: [PATCH 178/210] Break lines --- .github/workflows/dispatch.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 29a5323..ecc8eb1 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -58,7 +58,16 @@ on: default: false description: DEPLOY_TITILER_CMR -run-name: Dispatch to ${{ inputs.environment }} AUTH=${{ inputs.DEPLOY_AUTH }} BACKEND=${{ inputs.DEPLOY_BACKEND }} FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} SM2A=${{ inputs.DEPLOY_SM2A }} MONITORING=${{ inputs.DEPLOY_MONITORING }} TITILER_MULTIDIM=${{ inputs.DEPLOY_TITILER_MULTIDIM }} β›Ÿ +run-name: | + Dispatch to ${{ inputs.environment }} \ + AUTH=${{ inputs.DEPLOY_AUTH }} \ + BACKEND=${{ inputs.DEPLOY_BACKEND }} \ + FEATURES=${{ inputs.DEPLOY_FEATURES_API }} \ + ROUTES=${{ inputs.DEPLOY_ROUTES }} \ + SM2A=${{ inputs.DEPLOY_SM2A }} \ + MONITORING=${{ inputs.DEPLOY_MONITORING }} \ + TITILER_MULTIDIM=${{ inputs.DEPLOY_TITILER_MULTIDIM }} \ + TITILER_CMR=${{ inputs.DEPLOY_TITILER_CMR }} β›Ÿ jobs: check-environment: From 63f37173e14e98532f9b874615a305ba8887e799 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Thu, 22 May 2025 16:37:29 -0700 Subject: [PATCH 179/210] Remove line breaks --- .github/workflows/dispatch.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index ecc8eb1..689a2cc 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -59,14 +59,14 @@ on: description: DEPLOY_TITILER_CMR run-name: | - Dispatch to ${{ inputs.environment }} \ - AUTH=${{ inputs.DEPLOY_AUTH }} \ - BACKEND=${{ inputs.DEPLOY_BACKEND }} \ - FEATURES=${{ inputs.DEPLOY_FEATURES_API }} \ - ROUTES=${{ inputs.DEPLOY_ROUTES }} \ - SM2A=${{ inputs.DEPLOY_SM2A }} \ - MONITORING=${{ inputs.DEPLOY_MONITORING }} \ - TITILER_MULTIDIM=${{ inputs.DEPLOY_TITILER_MULTIDIM }} \ + Dispatch to ${{ inputs.environment }} + AUTH=${{ inputs.DEPLOY_AUTH }} + BACKEND=${{ inputs.DEPLOY_BACKEND }} + FEATURES=${{ inputs.DEPLOY_FEATURES_API }} + ROUTES=${{ inputs.DEPLOY_ROUTES }} + SM2A=${{ inputs.DEPLOY_SM2A }} + MONITORING=${{ inputs.DEPLOY_MONITORING }} + TITILER_MULTIDIM=${{ inputs.DEPLOY_TITILER_MULTIDIM }} TITILER_CMR=${{ inputs.DEPLOY_TITILER_CMR }} β›Ÿ jobs: From 852acc2727843302168e906afe01826472ca73a0 Mon Sep 17 00:00:00 2001 From: Paridhi <073bct525.paridhi@pcampus.edu.np> Date: Fri, 23 May 2025 10:10:57 -0500 Subject: [PATCH 180/210] change env variable names for s3 disaster recovery --- .github/workflows/deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5731fb7..e6cf019 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -391,9 +391,9 @@ jobs: with: env_file: ".env" aws_region: ${{ vars.AWS_REGION }} - allow_batch_replication: ${{ vars.ALLOW_BATCH_REPLICATION }} - source_bucket: ${{ vars.SOURCE_BUCKET_NAME }} - destination_bucket: ${{ vars.DESTINATION_BUCKET_NAME }} + allow_batch_replication: ${{ vars.VEDA_S3_DISASTER_RECOV_ALLOW_BATCH_REPLICATION }} + source_bucket: ${{ vars.VEDA_S3_DISASTER_RECOV_SOURCE_BUCKET_NAME }} + destination_bucket: ${{ vars.VEDA_S3_DISASTER_RECOV_DESTINATION_BUCKET_NAME }} env_aws_secret_name: ${{ vars.DEPLOYMENT_ENV_SECRET_NAME }} dir: "${{ env.DIRECTORY }}" script_path: "${{ github.workspace }}/scripts/generate_env_file.py" From 12e8424f5c4486664475c492df229b6756d5f39d Mon Sep 17 00:00:00 2001 From: Stephen Kilbourn Date: Tue, 29 Jul 2025 17:28:26 -0600 Subject: [PATCH 181/210] add integration test for next links --- .../test_stac_pagination_next_link.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 integration_test/test_stac_pagination_next_link.py diff --git a/integration_test/test_stac_pagination_next_link.py b/integration_test/test_stac_pagination_next_link.py new file mode 100644 index 0000000..2decc12 --- /dev/null +++ b/integration_test/test_stac_pagination_next_link.py @@ -0,0 +1,97 @@ +import requests +from dotenv import load_dotenv +import os +import pytest + +load_dotenv() + +def _get_link(obj: dict, rel: str) -> str: + """ + Helper function to find a specific link by its 'rel' type in a STAC object's "links" array. + """ + if not obj or not obj.get("links"): + return None + return next((link for link in obj.get("links") if link.get("rel") == rel), None) + +def test_collection_pagination_next_link_is_valid(): + """ + Validates that for collections with enough items to require pagination, + the 'next' link provided in the item list is a valid URL that returns a 200 OK status. + + This test fetches all collections and then individually checks their item lists for pagination links. + It focuses on the custom host configuration as it's the primary way + dynamically generated links are used by end-users. + """ + custom_host = os.getenv("VEDA_CUSTOM_HOST") + stac_root_path = os.getenv("VEDA_STAC_ROOT_PATH", "") + + if not custom_host: + pytest.skip("VEDA_CUSTOM_HOST environment variable not set. Skipping pagination test.") + + # Construct the URL to the main collections endpoint + collections_url = f"https://{custom_host}/{stac_root_path.strip('/')}/collections" + print(f"Starting pagination test. Fetching collections from: {collections_url}") + + # Fetch all collections + try: + collections_response = requests.get(collections_url) + collections_response.raise_for_status() # Fail fast if collections endpoint is down + collections = collections_response.json().get("collections", []) + except (requests.exceptions.RequestException, ValueError) as e: + pytest.fail(f"Could not fetch or parse collections from {collections_url}. Error: {e}") + + found_and_tested_a_next_link = False + + # Iterate through each collection to find one with pagination + for collection_summary in collections: + collection_id = collection_summary.get("id") + if not collection_id: + print("Skipping a collection that is missing an 'id'.") + continue + + # Get the link to the collection's items. The 'items' link is within the collection's own links. + items_link = _get_link(collection_summary, "items") + if not (items_link and items_link.get("href")): + print(f"Skipping collection '{collection_id}': No 'items' link found.") + continue + + items_url = items_link["href"] + print(f"Checking items for collection '{collection_id}' at: {items_url}") + + # Fetch the first page of items for the collection + try: + items_response = requests.get(items_url) + if items_response.status_code != 200: + print(f"Warning: Could not fetch items for '{collection_id}'. Status: {items_response.status_code}") + continue + items_json = items_response.json() + except (requests.exceptions.RequestException, ValueError) as e: + print(f"Warning: Could not fetch or parse items for '{collection_id}'. Error: {e}") + continue + + # Check if a 'next' link exists on the first page of items + next_link = _get_link(items_json, "next") + if next_link and next_link.get("href"): + next_url = next_link["href"] + print(f" - Found 'next' link: {next_url}") + + # Make a GET request to the 'next' URL + try: + next_page_response = requests.get(next_url) + + # Assert that the link is valid and returns a 200 OK status + assert next_page_response.status_code == 200, f"'next' link for {collection_id} failed with status {next_page_response.status_code}" + print(f" - Success! 'next' link returned status {next_page_response.status_code}.") + + found_and_tested_a_next_link = True + + # Once we've successfully tested one 'next' link, we can exit to keep the test fast. + break + except requests.exceptions.RequestException as e: + pytest.fail(f"Request for 'next' link URL {next_url} failed. Error: {e}") + else: + print(f" - No 'next' link found for '{collection_id}' (collection may have less than one page of items).") + + # Final assertion to ensure the test actually performed a validation. + # If this fails, it means no collections with pagination were found across the entire API. + assert found_and_tested_a_next_link, "Test finished without finding any collection with a 'next' link to validate." From 7202806302d916d457996d8a4856b00100b8f801 Mon Sep 17 00:00:00 2001 From: Stephen Kilbourn Date: Wed, 30 Jul 2025 13:35:26 -0600 Subject: [PATCH 182/210] some projects do not have next links, so do not fail if no next links --- integration_test/test_stac_pagination_next_link.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/integration_test/test_stac_pagination_next_link.py b/integration_test/test_stac_pagination_next_link.py index 2decc12..ebce5b4 100644 --- a/integration_test/test_stac_pagination_next_link.py +++ b/integration_test/test_stac_pagination_next_link.py @@ -40,8 +40,6 @@ def test_collection_pagination_next_link_is_valid(): except (requests.exceptions.RequestException, ValueError) as e: pytest.fail(f"Could not fetch or parse collections from {collections_url}. Error: {e}") - found_and_tested_a_next_link = False - # Iterate through each collection to find one with pagination for collection_summary in collections: collection_id = collection_summary.get("id") @@ -83,15 +81,10 @@ def test_collection_pagination_next_link_is_valid(): assert next_page_response.status_code == 200, f"'next' link for {collection_id} failed with status {next_page_response.status_code}" print(f" - Success! 'next' link returned status {next_page_response.status_code}.") - found_and_tested_a_next_link = True # Once we've successfully tested one 'next' link, we can exit to keep the test fast. break except requests.exceptions.RequestException as e: pytest.fail(f"Request for 'next' link URL {next_url} failed. Error: {e}") else: - print(f" - No 'next' link found for '{collection_id}' (collection may have less than one page of items).") - - # Final assertion to ensure the test actually performed a validation. - # If this fails, it means no collections with pagination were found across the entire API. - assert found_and_tested_a_next_link, "Test finished without finding any collection with a 'next' link to validate." + print(f" - No 'next' link found for '{collection_id}' (collection may have less than one page of items).") \ No newline at end of file From 2cab85899a8ce84ff24df4bd1715ed54a9c5da71 Mon Sep 17 00:00:00 2001 From: Stephen Kilbourn Date: Thu, 7 Aug 2025 13:13:22 -0600 Subject: [PATCH 183/210] add x-cache assertion for cloudfront error --- integration_test/test_stac_pagination_next_link.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/integration_test/test_stac_pagination_next_link.py b/integration_test/test_stac_pagination_next_link.py index ebce5b4..ed504f0 100644 --- a/integration_test/test_stac_pagination_next_link.py +++ b/integration_test/test_stac_pagination_next_link.py @@ -16,7 +16,8 @@ def _get_link(obj: dict, rel: str) -> str: def test_collection_pagination_next_link_is_valid(): """ Validates that for collections with enough items to require pagination, - the 'next' link provided in the item list is a valid URL that returns a 200 OK status. + the 'next' link provided in the item list is a valid URL that returns a 200 OK status + and does not contain a CloudFront cache error. This test fetches all collections and then individually checks their item lists for pagination links. It focuses on the custom host configuration as it's the primary way @@ -80,8 +81,11 @@ def test_collection_pagination_next_link_is_valid(): # Assert that the link is valid and returns a 200 OK status assert next_page_response.status_code == 200, f"'next' link for {collection_id} failed with status {next_page_response.status_code}" print(f" - Success! 'next' link returned status {next_page_response.status_code}.") - - + + x_cache_header = next_page_response.headers.get("x-cache", "").lower() + assert "error from cloudfront" not in x_cache_header, f"CloudFront error detected in x-cache header for {collection_id}: {x_cache_header}" + print(f" - Success! No CloudFront error found in x-cache header.") + # Once we've successfully tested one 'next' link, we can exit to keep the test fast. break except requests.exceptions.RequestException as e: From 3b154363bda06c08f9232937351d2c6a70cd032c Mon Sep 17 00:00:00 2001 From: ividito Date: Mon, 11 Aug 2025 13:13:10 -0700 Subject: [PATCH 184/210] Add rolling updates to deployment status branch --- .github/workflows/dispatch.yml | 8 ++ .../workflows/update_deployment_status.yml | 47 +++++++ scripts/update_deployment_status.py | 118 ++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 .github/workflows/update_deployment_status.yml create mode 100644 scripts/update_deployment_status.py diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 80d790d..1f4fe3e 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -103,3 +103,11 @@ jobs: DEPLOY_S3_DISASTER_RECOVERY: ${{ github.event.inputs.DEPLOY_S3_DISASTER_RECOVERY }} DEPLOY_TITILER_CMR: ${{ github.event.inputs.DEPLOY_TITILER_CMR }} secrets: inherit + + update-deployment-status: + name: Update Deployment Status + uses: "./.github/workflows/update_deployment_status.yml" + needs: deploy-veda-components + with: + environment: ${{ github.event.inputs.environment }} + secrets: inherit diff --git a/.github/workflows/update_deployment_status.yml b/.github/workflows/update_deployment_status.yml new file mode 100644 index 0000000..4d9008a --- /dev/null +++ b/.github/workflows/update_deployment_status.yml @@ -0,0 +1,47 @@ +name: Update Deployment Status + +on: + workflow_call: + inputs: + environment: + required: true + type: string + +jobs: + update-status-md: + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + + steps: + - name: Checkout deployment-state branch + uses: actions/checkout@v4 + with: + ref: deployment-state + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.x + + - name: Run update script + run: | + python scripts/update_deployment_state.py \ + --env "${{ inputs.environment }}" \ + --auth "${{ vars.VEDA_AUTH_GIT_REF }}" \ + --backend "${{ vars.VEDA_BACKEND_GIT_REF }}" \ + --features_api "${{ vars.VEDA_FEATURES_API_GIT_REF }}" \ + --routes "${{ vars.VEDA_ROUTES_GIT_REF }}" \ + --sm2a "${{ vars.VEDA_SM2A_GIT_REF }}" \ + --monitoring "${{ vars.VEDA_MONITORING_GIT_REF }}" \ + --titiler_multidim "${{ vars.VEDA_TITILER_MULTIDIM_GIT_REF }}" \ + --s3_dr "${{ vars.VEDA_S3_DISASTER_RECOVERY_GIT_REF }}" \ + --titiler_cmr "${{ vars.VEDA_TITILER_CMR_GIT_REF }}" + + - name: Commit and push updated deployment-status.md + run: | + git config user.name "github-actions" + git config user.email "github-actions@github.com" + git add deployment-status.md + git commit -m "Update deployment refs for ${{ inputs.environment }}" + git push origin deployment-state diff --git a/scripts/update_deployment_status.py b/scripts/update_deployment_status.py new file mode 100644 index 0000000..c41cc84 --- /dev/null +++ b/scripts/update_deployment_status.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +import argparse, os, re +from collections import defaultdict, OrderedDict +from datetime import datetime, timezone + + +DEPLOYMENT_FILE = "deployment-status.md" + +# component_id, DEPLOY_* env, VEDA_*_GIT_REF env, display name in MD, repo URL +COMPONENTS = [ + ("auth", "DEPLOY_AUTH", "VEDA_AUTH_GIT_REF", "AUTH", "https://github.com/NASA-IMPACT/veda-auth"), + ("backend", "DEPLOY_BACKEND", "VEDA_BACKEND_GIT_REF", "BACKEND", "https://github.com/NASA-IMPACT/veda-backend"), + ("features_api", "DEPLOY_FEATURES_API", "VEDA_FEATURES_API_GIT_REF", "FEATURES_API", "https://github.com/NASA-IMPACT/veda-features-api-cdk"), + ("routes", "DEPLOY_ROUTES", "VEDA_ROUTES_GIT_REF", "ROUTES", "https://github.com/NASA-IMPACT/veda-routes"), + ("sm2a", "DEPLOY_SM2A", "VEDA_SM2A_GIT_REF", "SM2A", "https://github.com/NASA-IMPACT/veda-sm2a"), + ("monitoring", "DEPLOY_MONITORING", "VEDA_MONITORING_GIT_REF", "MONITORING", "https://github.com/NASA-IMPACT/veda-monitoring"), + ("titiler_multidim", "DEPLOY_TITILER_MULTIDIM", "VEDA_TITILER_MULTIDIM_GIT_REF", "TITILER_MULTIDIM", "https://github.com/developmentseed/titiler-multidim"), + ("s3_dr", "DEPLOY_S3_DISASTER_RECOVERY", "VEDA_S3_DISASTER_RECOVERY_GIT_REF", "S3_DR", "https://github.com/NASA-IMPACT/s3-disaster-recovery"), + ("titiler_cmr", "DEPLOY_TITILER_CMR", "VEDA_TITILER_CMR_GIT_REF", "TITILER_CMR", "https://github.com/developmentseed/titiler-cmr"), +] + +ROW_RE = re.compile(r'^\|\s*([^|]+?)\s*\|\s*([^|]+?)\s*\|\s*([^|]+?)\s*\|$') +ENV_RE = re.compile(r'^##\s+(.+)\s*$') + +def load_state(path: str) -> dict: + """Parse existing markdown into { env: { DISPLAY_NAME: ref } }.""" + state = defaultdict(dict) + if not os.path.exists(path): + return state + + current_env = None + with open(path, "r", encoding="utf-8") as f: + for raw in f: + line = raw.rstrip("\n") + + m_env = ENV_RE.match(line) + if m_env: + current_env = m_env.group(1).strip() + continue + + if not current_env or not line.startswith("|"): + continue + + m_row = ROW_RE.match(line) + if not m_row: + continue + + c1 = m_row.group(1).strip() + c2 = m_row.group(2).strip() + c3 = m_row.group(3).strip() + + if c1.lower() == "component": + continue + if set(c1) == {"-"} and set(c2) == {"-"}: + continue + + state[current_env][c1] = (c2, c3) + return state + +def make_ref_link(repo_url: str, ref: str) -> str: + if not ref: + return "" + return f"[{ref}]({repo_url}/tree/{ref})" + +def write_state(path: str, state: dict): + ordered_envs = sorted(state.keys()) + with open(path, "w", encoding="utf-8") as f: + f.write("# Deployment Status\n\nIf a component or environment is not listed, it has not yet been deployed through veda-deploy.\n\n") + for env in ordered_envs: + f.write(f"## {env}\n") + f.write("| Component | Git Ref | Updated (UTC) |\n") + f.write("|-----------|---------|---------------|\n") + for _, _, _, disp, repo_url in COMPONENTS: + entry = state[env].get(disp, ("","")) + print(f"Processing {disp} for {env}: {entry}") + ref, upd = entry + ref_link = ref if ref.startswith("[") else make_ref_link(repo_url, ref) + f.write(f"| {disp} | {ref_link} | {upd} |\n") + f.write("\n") + +def collect_updates_from_env() -> dict: + """Return { DISPLAY_NAME: ref } for components that should be updated.""" + updates = {} + ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S') + for _, deploy_env, ref_env, display, _ in COMPONENTS: + deploy_flag = os.getenv(deploy_env, "") + ref_val = os.getenv(ref_env, "") + if deploy_flag and ref_val.strip(): + updates[display] = (ref_val.strip(), ts) + return updates + +def main(): + print("Updating deployment status...") + ap = argparse.ArgumentParser(description="Update deployment-status.md with current refs.") + ap.add_argument("--env", required=True, help="Target environment name (must match your GH environment)") + ap.add_argument("--file", default=DEPLOYMENT_FILE, help="Path to deployment status markdown") + args = ap.parse_args() + + state = load_state(args.file) + print(f"Loaded existing state for {args.env}: {state.get(args.env, {})}") + # Ensure env exists in state even if empty + _ = state[args.env] + + updates = collect_updates_from_env() + if not updates: + # Nothing to do; write existing state back unchanged (no-op) + write_state(args.file, state) + return + + # Apply updates for selected components only + for display, ref in updates.items(): + state[args.env][display] = ref + + write_state(args.file, state) + +if __name__ == "__main__": + print("Starting deployment status update script...") + main() \ No newline at end of file From f31efa72f6c918e914623c397345933aeb894f15 Mon Sep 17 00:00:00 2001 From: ividito Date: Fri, 15 Aug 2025 16:49:13 -0700 Subject: [PATCH 185/210] Correct script name --- .github/workflows/update_deployment_status.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_deployment_status.yml b/.github/workflows/update_deployment_status.yml index 4d9008a..37368df 100644 --- a/.github/workflows/update_deployment_status.yml +++ b/.github/workflows/update_deployment_status.yml @@ -26,7 +26,7 @@ jobs: - name: Run update script run: | - python scripts/update_deployment_state.py \ + python scripts/update_deployment_status.py \ --env "${{ inputs.environment }}" \ --auth "${{ vars.VEDA_AUTH_GIT_REF }}" \ --backend "${{ vars.VEDA_BACKEND_GIT_REF }}" \ From 66afe12ae07c7deaaff9a4c018e8d480f4267864 Mon Sep 17 00:00:00 2001 From: ividito Date: Fri, 15 Aug 2025 16:52:25 -0700 Subject: [PATCH 186/210] Add debug step --- .github/workflows/update_deployment_status.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/update_deployment_status.yml b/.github/workflows/update_deployment_status.yml index 37368df..fcde61d 100644 --- a/.github/workflows/update_deployment_status.yml +++ b/.github/workflows/update_deployment_status.yml @@ -24,6 +24,9 @@ jobs: with: python-version: 3.x + - name: DEBUG show current directory + run: pwd; ls -la + - name: Run update script run: | python scripts/update_deployment_status.py \ From 7245aea8a384702e4a0335005ce20cf77c025667 Mon Sep 17 00:00:00 2001 From: ividito Date: Fri, 15 Aug 2025 19:56:28 -0700 Subject: [PATCH 187/210] Remove debug step, deprecated arguments --- .github/workflows/update_deployment_status.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/workflows/update_deployment_status.yml b/.github/workflows/update_deployment_status.yml index fcde61d..c066e3d 100644 --- a/.github/workflows/update_deployment_status.yml +++ b/.github/workflows/update_deployment_status.yml @@ -24,23 +24,10 @@ jobs: with: python-version: 3.x - - name: DEBUG show current directory - run: pwd; ls -la - - name: Run update script run: | python scripts/update_deployment_status.py \ - --env "${{ inputs.environment }}" \ - --auth "${{ vars.VEDA_AUTH_GIT_REF }}" \ - --backend "${{ vars.VEDA_BACKEND_GIT_REF }}" \ - --features_api "${{ vars.VEDA_FEATURES_API_GIT_REF }}" \ - --routes "${{ vars.VEDA_ROUTES_GIT_REF }}" \ - --sm2a "${{ vars.VEDA_SM2A_GIT_REF }}" \ - --monitoring "${{ vars.VEDA_MONITORING_GIT_REF }}" \ - --titiler_multidim "${{ vars.VEDA_TITILER_MULTIDIM_GIT_REF }}" \ - --s3_dr "${{ vars.VEDA_S3_DISASTER_RECOVERY_GIT_REF }}" \ - --titiler_cmr "${{ vars.VEDA_TITILER_CMR_GIT_REF }}" - + --env "${{ inputs.environment }}" - name: Commit and push updated deployment-status.md run: | git config user.name "github-actions" From 2f749c5e07e0a4fa425124c16141d9a3f0f9bbe6 Mon Sep 17 00:00:00 2001 From: ividito Date: Fri, 15 Aug 2025 20:12:53 -0700 Subject: [PATCH 188/210] Add write permission for repo --- .github/workflows/update_deployment_status.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/update_deployment_status.yml b/.github/workflows/update_deployment_status.yml index c066e3d..cc80d0f 100644 --- a/.github/workflows/update_deployment_status.yml +++ b/.github/workflows/update_deployment_status.yml @@ -7,6 +7,9 @@ on: required: true type: string +permissions: + contents: write + jobs: update-status-md: runs-on: ubuntu-latest From d26f0af9fd48180a7a559143a4c5f8d83b0e7eae Mon Sep 17 00:00:00 2001 From: ividito Date: Sat, 16 Aug 2025 09:55:03 -0700 Subject: [PATCH 189/210] Expand permission boundary --- .github/workflows/dispatch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 1f4fe3e..71db6cf 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -2,7 +2,7 @@ name: Dispatch β›Ÿ permissions: id-token: write - contents: read + contents: write issues: write on: From faaf9d96c23525488bf223102261166c2a74fed3 Mon Sep 17 00:00:00 2001 From: ividito Date: Mon, 18 Aug 2025 12:40:35 -0700 Subject: [PATCH 190/210] Set env at top level of workflow --- .github/workflows/dispatch.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 71db6cf..8f476b3 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -69,6 +69,17 @@ run-name: | TITILER_MULTIDIM=${{ inputs.DEPLOY_TITILER_MULTIDIM }} TITILER_CMR=${{ inputs.DEPLOY_TITILER_CMR }} β›Ÿ +env: + DEPLOY_AUTH: ${{ github.event.inputs.DEPLOY_AUTH}} + DEPLOY_BACKEND: ${{ github.event.inputs.DEPLOY_BACKEND }} + DEPLOY_FEATURES_API: ${{ github.event.inputs.DEPLOY_FEATURES_API }} + DEPLOY_ROUTES: ${{ github.event.inputs.DEPLOY_ROUTES }} + DEPLOY_SM2A: ${{ github.event.inputs.DEPLOY_SM2A }} + DEPLOY_MONITORING: ${{ github.event.inputs.DEPLOY_MONITORING }} + DEPLOY_TITILER_MULTIDIM: ${{ github.event.inputs.DEPLOY_TITILER_MULTIDIM }} + DEPLOY_S3_DISASTER_RECOVERY: ${{ github.event.inputs.DEPLOY_S3_DISASTER_RECOVERY }} + DEPLOY_TITILER_CMR: ${{ github.event.inputs.DEPLOY_TITILER_CMR }} + jobs: check-environment: runs-on: ubuntu-latest From 863200e51d1e0e94c15feb9fc49f3988f61a4d89 Mon Sep 17 00:00:00 2001 From: ividito Date: Mon, 18 Aug 2025 12:43:22 -0700 Subject: [PATCH 191/210] Include git refs in environment --- .github/workflows/update_deployment_status.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/update_deployment_status.yml b/.github/workflows/update_deployment_status.yml index cc80d0f..98487d1 100644 --- a/.github/workflows/update_deployment_status.yml +++ b/.github/workflows/update_deployment_status.yml @@ -31,6 +31,16 @@ jobs: run: | python scripts/update_deployment_status.py \ --env "${{ inputs.environment }}" + env: + VEDA_AUTH_GIT_REF: ${{ vars.VEDA_AUTH_GIT_REF }} + VEDA_BACKEND_GIT_REF: ${{ vars.VEDA_BACKEND_GIT_REF }} + VEDA_FEATURES_API_GIT_REF: ${{ vars.VEDA_FEATURES_API_GIT_REF }} + VEDA_ROUTES_GIT_REF: ${{ vars.VEDA_ROUTES_GIT_REF }} + VEDA_SM2A_GIT_REF: ${{ vars.VEDA_SM2A_GIT_REF }} + VEDA_MONITORING_GIT_REF: ${{ vars.VEDA_MONITORING_GIT_REF }} + VEDA_TITILER_MULTIDIM_GIT_REF: ${{ vars.VEDA_TITILER_MULTIDIM_GIT_REF }} + VEDA_S3_DISASTER_RECOVERY_GIT_REF: ${{ vars.VEDA_S3_DISASTER_RECOVERY_GIT_REF }} + VEDA_TITILER_CMR_GIT_REF: ${{ vars.VEDA_TITILER_CMR_GIT_REF }} - name: Commit and push updated deployment-status.md run: | git config user.name "github-actions" From 2fb249ae962562912c427cf3246a0e7c236c1876 Mon Sep 17 00:00:00 2001 From: ividito Date: Mon, 18 Aug 2025 12:59:27 -0700 Subject: [PATCH 192/210] Debug logs --- scripts/update_deployment_status.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/update_deployment_status.py b/scripts/update_deployment_status.py index c41cc84..2ac8a67 100644 --- a/scripts/update_deployment_status.py +++ b/scripts/update_deployment_status.py @@ -83,8 +83,10 @@ def collect_updates_from_env() -> dict: updates = {} ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S') for _, deploy_env, ref_env, display, _ in COMPONENTS: + print(f"Checking {display} for environment variables: {deploy_env}, {ref_env}") deploy_flag = os.getenv(deploy_env, "") ref_val = os.getenv(ref_env, "") + print(f" {deploy_env} = {deploy_flag}, {ref_env} = {ref_val}") if deploy_flag and ref_val.strip(): updates[display] = (ref_val.strip(), ts) return updates From 180ab708561cae7f223423d212d033a812f2bb47 Mon Sep 17 00:00:00 2001 From: ividito Date: Mon, 18 Aug 2025 13:01:14 -0700 Subject: [PATCH 193/210] More environment variables --- .github/workflows/update_deployment_status.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/update_deployment_status.yml b/.github/workflows/update_deployment_status.yml index 98487d1..1bf8c52 100644 --- a/.github/workflows/update_deployment_status.yml +++ b/.github/workflows/update_deployment_status.yml @@ -41,6 +41,15 @@ jobs: VEDA_TITILER_MULTIDIM_GIT_REF: ${{ vars.VEDA_TITILER_MULTIDIM_GIT_REF }} VEDA_S3_DISASTER_RECOVERY_GIT_REF: ${{ vars.VEDA_S3_DISASTER_RECOVERY_GIT_REF }} VEDA_TITILER_CMR_GIT_REF: ${{ vars.VEDA_TITILER_CMR_GIT_REF }} + DEPLOY_BACKEND: ${{ github.event.inputs.DEPLOY_BACKEND }} + DEPLOY_AUTH: ${{ github.event.inputs.DEPLOY_AUTH }} + DEPLOY_FEATURES_API: ${{ github.event.inputs.DEPLOY_FEATURES_API }} + DEPLOY_ROUTES: ${{ github.event.inputs.DEPLOY_ROUTES }} + DEPLOY_SM2A: ${{ github.event.inputs.DEPLOY_SM2A }} + DEPLOY_MONITORING: ${{ github.event.inputs.DEPLOY_MONITORING }} + DEPLOY_TITILER_MULTIDIM: ${{ github.event.inputs.DEPLOY_TITILER_MULTIDIM }} + DEPLOY_S3_DISASTER_RECOVERY: ${{ github.event.inputs.DEPLOY_S3_DISASTER_RECOVERY }} + DEPLOY_TITILER_CMR: ${{ github.event.inputs.DEPLOY_TITILER_CMR }} - name: Commit and push updated deployment-status.md run: | git config user.name "github-actions" From 23e49a81920e0a31261bc07315f968bf2ace0e8b Mon Sep 17 00:00:00 2001 From: ividito Date: Mon, 18 Aug 2025 14:26:54 -0700 Subject: [PATCH 194/210] Another batch of debug lines --- .github/workflows/deploy.yml | 18 +++++++++--------- scripts/update_deployment_status.py | 1 + 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c07d655..255c56a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -14,47 +14,47 @@ on: required: true description: Environment to deploy to DEPLOY_AUTH: - type: string + type: boolean required: true default: false description: DEPLOY_AUTH DEPLOY_BACKEND: - type: string + type: boolean required: true default: true description: DEPLOY_BACKEND DEPLOY_FEATURES_API: - type: string + type: boolean required: true default: false description: DEPLOY_FEATURES_API DEPLOY_ROUTES: - type: string + type: boolean required: true default: false description: DEPLOY_ROUTES DEPLOY_SM2A: - type: string + type: boolean required: true default: true description: DEPLOY_SM2A DEPLOY_MONITORING: - type: string + type: boolean required: true default: false description: DEPLOY_MONITORING DEPLOY_TITILER_MULTIDIM: - type: string + type: boolean required: true default: false description: DEPLOY_TITILER_MULTIDIM DEPLOY_S3_DISASTER_RECOVERY: - type: string + type: boolean required: true default: false description: DEPLOY_S3_DISASTER_RECOVERY DEPLOY_TITILER_CMR: - type: string + type: boolean required: true default: false description: DEPLOY_TITILER_CMR diff --git a/scripts/update_deployment_status.py b/scripts/update_deployment_status.py index 2ac8a67..ee71d08 100644 --- a/scripts/update_deployment_status.py +++ b/scripts/update_deployment_status.py @@ -104,6 +104,7 @@ def main(): _ = state[args.env] updates = collect_updates_from_env() + print(f"Collected updates: {updates}") if not updates: # Nothing to do; write existing state back unchanged (no-op) write_state(args.file, state) From 4d590bcd06e5b881bc064f2445c669f047f79f34 Mon Sep 17 00:00:00 2001 From: ividito Date: Mon, 18 Aug 2025 14:30:17 -0700 Subject: [PATCH 195/210] Revert string->boolean cast --- .github/workflows/deploy.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 255c56a..c07d655 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -14,47 +14,47 @@ on: required: true description: Environment to deploy to DEPLOY_AUTH: - type: boolean + type: string required: true default: false description: DEPLOY_AUTH DEPLOY_BACKEND: - type: boolean + type: string required: true default: true description: DEPLOY_BACKEND DEPLOY_FEATURES_API: - type: boolean + type: string required: true default: false description: DEPLOY_FEATURES_API DEPLOY_ROUTES: - type: boolean + type: string required: true default: false description: DEPLOY_ROUTES DEPLOY_SM2A: - type: boolean + type: string required: true default: true description: DEPLOY_SM2A DEPLOY_MONITORING: - type: boolean + type: string required: true default: false description: DEPLOY_MONITORING DEPLOY_TITILER_MULTIDIM: - type: boolean + type: string required: true default: false description: DEPLOY_TITILER_MULTIDIM DEPLOY_S3_DISASTER_RECOVERY: - type: boolean + type: string required: true default: false description: DEPLOY_S3_DISASTER_RECOVERY DEPLOY_TITILER_CMR: - type: boolean + type: string required: true default: false description: DEPLOY_TITILER_CMR From 6901d2c367a8b0c42e4e2064186be5dc703626ea Mon Sep 17 00:00:00 2001 From: ividito Date: Mon, 18 Aug 2025 15:04:30 -0700 Subject: [PATCH 196/210] Force flag to boolean --- scripts/update_deployment_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update_deployment_status.py b/scripts/update_deployment_status.py index ee71d08..fd261e9 100644 --- a/scripts/update_deployment_status.py +++ b/scripts/update_deployment_status.py @@ -87,7 +87,7 @@ def collect_updates_from_env() -> dict: deploy_flag = os.getenv(deploy_env, "") ref_val = os.getenv(ref_env, "") print(f" {deploy_env} = {deploy_flag}, {ref_env} = {ref_val}") - if deploy_flag and ref_val.strip(): + if deploy_flag == "true" and ref_val.strip(): # this is weird but GHA won't let the flag be cast as a boolean updates[display] = (ref_val.strip(), ts) return updates From 2ae4271f635a6a596b2ccfdbe47b9c5692ac4a70 Mon Sep 17 00:00:00 2001 From: ividito Date: Mon, 25 Aug 2025 16:28:43 -0700 Subject: [PATCH 197/210] Update SM2A git ref name --- .github/workflows/update_deployment_status.yml | 2 +- scripts/update_deployment_status.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update_deployment_status.yml b/.github/workflows/update_deployment_status.yml index 1bf8c52..1d14cdc 100644 --- a/.github/workflows/update_deployment_status.yml +++ b/.github/workflows/update_deployment_status.yml @@ -36,7 +36,7 @@ jobs: VEDA_BACKEND_GIT_REF: ${{ vars.VEDA_BACKEND_GIT_REF }} VEDA_FEATURES_API_GIT_REF: ${{ vars.VEDA_FEATURES_API_GIT_REF }} VEDA_ROUTES_GIT_REF: ${{ vars.VEDA_ROUTES_GIT_REF }} - VEDA_SM2A_GIT_REF: ${{ vars.VEDA_SM2A_GIT_REF }} + VEDA_SM2A_DATA_AIRFLOW_GIT_REF: ${{ vars.VEDA_SM2A_DATA_AIRFLOW_GIT_REF }} VEDA_MONITORING_GIT_REF: ${{ vars.VEDA_MONITORING_GIT_REF }} VEDA_TITILER_MULTIDIM_GIT_REF: ${{ vars.VEDA_TITILER_MULTIDIM_GIT_REF }} VEDA_S3_DISASTER_RECOVERY_GIT_REF: ${{ vars.VEDA_S3_DISASTER_RECOVERY_GIT_REF }} diff --git a/scripts/update_deployment_status.py b/scripts/update_deployment_status.py index fd261e9..f76272b 100644 --- a/scripts/update_deployment_status.py +++ b/scripts/update_deployment_status.py @@ -12,7 +12,7 @@ ("backend", "DEPLOY_BACKEND", "VEDA_BACKEND_GIT_REF", "BACKEND", "https://github.com/NASA-IMPACT/veda-backend"), ("features_api", "DEPLOY_FEATURES_API", "VEDA_FEATURES_API_GIT_REF", "FEATURES_API", "https://github.com/NASA-IMPACT/veda-features-api-cdk"), ("routes", "DEPLOY_ROUTES", "VEDA_ROUTES_GIT_REF", "ROUTES", "https://github.com/NASA-IMPACT/veda-routes"), - ("sm2a", "DEPLOY_SM2A", "VEDA_SM2A_GIT_REF", "SM2A", "https://github.com/NASA-IMPACT/veda-sm2a"), + ("sm2a", "DEPLOY_SM2A", "VEDA_SM2A_DATA_AIRFLOW_GIT_REF", "SM2A", "https://github.com/NASA-IMPACT/veda-sm2a"), ("monitoring", "DEPLOY_MONITORING", "VEDA_MONITORING_GIT_REF", "MONITORING", "https://github.com/NASA-IMPACT/veda-monitoring"), ("titiler_multidim", "DEPLOY_TITILER_MULTIDIM", "VEDA_TITILER_MULTIDIM_GIT_REF", "TITILER_MULTIDIM", "https://github.com/developmentseed/titiler-multidim"), ("s3_dr", "DEPLOY_S3_DISASTER_RECOVERY", "VEDA_S3_DISASTER_RECOVERY_GIT_REF", "S3_DR", "https://github.com/NASA-IMPACT/s3-disaster-recovery"), From bd8c6a00e675cef3f58456e160127757d5e04693 Mon Sep 17 00:00:00 2001 From: ividito Date: Wed, 27 Aug 2025 16:27:23 -0700 Subject: [PATCH 198/210] Set test timeout --- .github/workflows/deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c07d655..dd6a10f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -512,6 +512,7 @@ jobs: test-deployment: name: Test Deployment πŸ‘¨πŸ»β€πŸ”¬ runs-on: ubuntu-latest + timeout-minutes: 15 # Normally takes ~30s needs: [ deploy-veda-backend ] env: DIRECTORY: integration_test From bd35935dc0d3ad82201f052892b9cd097f8f91da Mon Sep 17 00:00:00 2001 From: aliziel <21992503+aliziel@users.noreply.github.com> Date: Mon, 16 Mar 2026 11:37:22 -0700 Subject: [PATCH 199/210] ci: add terraform plan to cicd --- .github/workflows/diff.yml | 62 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/diff.yml diff --git a/.github/workflows/diff.yml b/.github/workflows/diff.yml new file mode 100644 index 0000000..8dda9f7 --- /dev/null +++ b/.github/workflows/diff.yml @@ -0,0 +1,62 @@ +name: Diff Infrastructure Changes +description: View IaC changes against current state, not applied/deployed + +permissions: + id-token: write + contents: write + issues: write + +on: + workflow_dispatch: + inputs: + environment: + type: environment + required: true + description: Environment to diff + IaC: + required: true + type: choice + options: + - SM2A + +jobs: + plan-sm2a: + name: '`terraform plan` sm2a' + runs-on: ubuntu-latest + env: + DIRECTORY: veda-data-airflow + AWS_REGION: "us-west-2" + ENVIRONMENT: ${{ inputs.environment }} + if: ${{ inputs.IaC == 'SM2A' }} + environment: ${{ inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + lfs: "true" + submodules: "false" + + - name: Checkout veda-data-airflow submodule + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: "NASA-IMPACT/${{ env.DIRECTORY }}" + path: ${{ env.DIRECTORY }} + ref: ${{ vars.VEDA_SM2A_DATA_AIRFLOW_GIT_REF || 'main'}} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 + with: + role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} + role-session-name: "gh-${{ env.ENVIRONMENT }}-airflow-sm2a-deployment" + aws-region: "${{ env.AWS_REGION }}" + + - name: Run `terraform plan` + id: plan_sm2a + uses: "./veda-data-airflow/.github/actions/terraform-deploy-sm2a" + with: + env-file: ".env" + env_aws_secret_name: ${{ vars.SM2A_ENVS_DEPLOYMENT_SECRET_NAME }} + dir: "${{ env.DIRECTORY }}" + script_path: "${{ github.workspace }}/scripts/generate_env_file.py" + run_command: plan From d99398256e48b2961cb1b7b3d88a7c563d5c7704 Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Mon, 30 Mar 2026 11:38:25 -0700 Subject: [PATCH 200/210] fix: remove prefix setting for veda-routes job --- .github/workflows/deploy.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 87e30d5..db27e94 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -407,11 +407,10 @@ jobs: run: | python3 "${{ github.workspace }}/scripts/update_secret_with_inputs.py" \ --secret-id ${{ vars.DEPLOYMENT_ENV_ROUTES_SECRET_NAME }} \ - --prefix=VEDA_ \ --raster_api_url=${{ needs.deploy-veda-backend.outputs.raster_api_url }} \ --ingest_api_url=${{ needs.deploy-veda-backend.outputs.ingest_api_url }} \ --stac_api_url=${{ needs.deploy-veda-backend.outputs.stac_api_url }} \ - --stac_browser_bucket_name=${{ needs.deploy-veda-backend.outputs.stac_browser_bucket_name }} + --stac_browser_bucket_name=${{ needs.deploy-veda-backend.outputs.stac_browser_bucket_name }} - name: Run deployment uses: "./veda-routes/.github/actions/cdk-deploy" From c31ef053b8fcee6e3b9047bd9536a5feec718871 Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Mon, 30 Mar 2026 12:31:55 -0700 Subject: [PATCH 201/210] fix: rename stac_browser_bucket_name to stac_browser_bucket --- .github/workflows/deploy.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index db27e94..bacddc2 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -160,15 +160,15 @@ jobs: stac_api_url=$(jq '.[keys_unsorted[0]].stacapiurl' ${HOME}/cdk-outputs.json) echo "stac_api_url=$stac_api_url" >> $GITHUB_OUTPUT - stac_browser_bucket_name=$(jq '.[keys_unsorted[0]].stacbrowserbucketname' ${HOME}/cdk-outputs.json) - echo "stac_browser_bucket_name=$stac_browser_bucket_name" >> $GITHUB_OUTPUT + stac_browser_bucket=$(jq '.[keys_unsorted[0]].stacbrowserbucketname' ${HOME}/cdk-outputs.json) + echo "stac_browser_bucket=$stac_browser_bucket" >> $GITHUB_OUTPUT outputs: backend_stack_name: ${{ steps.get_backend_stack.outputs.backend_stackname }} raster_api_url: ${{ steps.get_backend_stack.outputs.raster_api_url }} ingest_api_url: ${{ steps.get_backend_stack.outputs.ingest_api_url }} stac_api_url: ${{ steps.get_backend_stack.outputs.stac_api_url }} - stac_browser_bucket_name: ${{ steps.get_backend_stack.outputs.stac_browser_bucket_name }} + stac_browser_bucket: ${{ steps.get_backend_stack.outputs.stac_browser_bucket }} deploy-veda-data-airflow-sm2a: name: deploy VEDA data airflow SM2A πŸ›Έ @@ -410,7 +410,7 @@ jobs: --raster_api_url=${{ needs.deploy-veda-backend.outputs.raster_api_url }} \ --ingest_api_url=${{ needs.deploy-veda-backend.outputs.ingest_api_url }} \ --stac_api_url=${{ needs.deploy-veda-backend.outputs.stac_api_url }} \ - --stac_browser_bucket_name=${{ needs.deploy-veda-backend.outputs.stac_browser_bucket_name }} + --stac_browser_bucket=${{ needs.deploy-veda-backend.outputs.stac_browser_bucket }} - name: Run deployment uses: "./veda-routes/.github/actions/cdk-deploy" From 82191f24c1cda3d11f3792e19fc305853e3f34d0 Mon Sep 17 00:00:00 2001 From: aliziel <21992503+aliziel@users.noreply.github.com> Date: Thu, 2 Apr 2026 09:19:51 -0700 Subject: [PATCH 202/210] Apply suggestions from code review Co-authored-by: Jennifer Tran <12633533+botanical@users.noreply.github.com> --- .github/workflows/diff.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/diff.yml b/.github/workflows/diff.yml index 8dda9f7..c59c0a6 100644 --- a/.github/workflows/diff.yml +++ b/.github/workflows/diff.yml @@ -1,10 +1,9 @@ name: Diff Infrastructure Changes -description: View IaC changes against current state, not applied/deployed permissions: id-token: write - contents: write - issues: write + contents: read + issues: read on: workflow_dispatch: From e46dea6164b1ab990b1b629d76cc06efc17351cc Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Fri, 17 Apr 2026 11:00:20 -0700 Subject: [PATCH 203/210] chore: upgrade to use node 22 --- .github/workflows/deploy.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e363611..46ffcf7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -561,10 +561,10 @@ jobs: repository: NASA-IMPACT/veda-config ref: add-playwright - - name: Use Node.js 16 - uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 #v3.8.2 + - name: Use Node.js 22 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: - node-version: 16 + node-version: 22 - name: Run veda setup run: ./.veda/setup From 4119868ff543a6dd36e577198faebd8d50c6fa83 Mon Sep 17 00:00:00 2001 From: Sandra Hoang Date: Mon, 11 May 2026 10:07:31 -0400 Subject: [PATCH 204/210] remove veda_auth and deps --- .github/workflows/deploy.yml | 52 ------------------- .github/workflows/dispatch.yml | 8 --- .../workflows/update_deployment_status.yml | 2 - .gitmodules | 3 -- README.md | 2 - scripts/update_deployment_status.py | 1 - 6 files changed, 68 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e363611..9b8af7b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -13,11 +13,6 @@ on: type: string required: true description: Environment to deploy to - DEPLOY_AUTH: - type: string - required: true - default: false - description: DEPLOY_AUTH DEPLOY_BACKEND: type: string required: true @@ -67,53 +62,6 @@ run-name: Deploy to ${{ inputs.environment }} environment by @${{ github.actor } jobs: - deploy-veda-auth: - name: Deploy VEDA auth πŸ” - runs-on: ubuntu-latest - env: - DIRECTORY: veda-auth - ENVIRONMENT: ${{ inputs.environment }} - environment: ${{ inputs.environment }} - if: ${{ inputs.DEPLOY_AUTH == 'true' }} - - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - lfs: "true" - submodules: "false" - - - name: Checkout veda-auth submodule - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: "NASA-IMPACT/${{ env.DIRECTORY }}" - path: ${{ env.DIRECTORY }} - ref: ${{ vars.VEDA_AUTH_GIT_REF || 'main'}} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 #v4.1.0 - with: - role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }} - role-session-name: "gh-${{ env.ENVIRONMENT }}-auth-deployment" - aws-region: "us-west-2" - - - name: Run auth deployment - id: deploy_auth_stack - uses: "./veda-auth/.github/actions/cdk-deploy" - with: - dir: "${{ env.DIRECTORY }}" - env_aws_secret_name: ${{ vars.VEDA_AUTH_DEPLOYMENT_SECRET || vars.DEPLOYMENT_ENV_SECRET_NAME }} - - - name: Get Auth Stack Name - id: get_auth_stack - shell: bash - run: | - stack=$(jq 'keys_unsorted[0]' ${HOME}/cdk-outputs.json) - echo "auth_stackname=$stack" >> $GITHUB_OUTPUT - - outputs: - auth_stack_name: ${{ steps.get_auth_stack.outputs.auth_stackname }} - deploy-veda-backend: name: Deploy VEDA backend βš™οΈ runs-on: ubuntu-latest diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 8f476b3..6513309 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -12,11 +12,6 @@ on: type: environment required: true description: Environment to deploy to - DEPLOY_AUTH: - type: boolean - required: true - default: false - description: DEPLOY_AUTH DEPLOY_BACKEND: type: boolean required: true @@ -60,7 +55,6 @@ on: run-name: | Dispatch to ${{ inputs.environment }} - AUTH=${{ inputs.DEPLOY_AUTH }} BACKEND=${{ inputs.DEPLOY_BACKEND }} FEATURES=${{ inputs.DEPLOY_FEATURES_API }} ROUTES=${{ inputs.DEPLOY_ROUTES }} @@ -70,7 +64,6 @@ run-name: | TITILER_CMR=${{ inputs.DEPLOY_TITILER_CMR }} β›Ÿ env: - DEPLOY_AUTH: ${{ github.event.inputs.DEPLOY_AUTH}} DEPLOY_BACKEND: ${{ github.event.inputs.DEPLOY_BACKEND }} DEPLOY_FEATURES_API: ${{ github.event.inputs.DEPLOY_FEATURES_API }} DEPLOY_ROUTES: ${{ github.event.inputs.DEPLOY_ROUTES }} @@ -104,7 +97,6 @@ jobs: needs: check-environment with: environment: ${{ github.event.inputs.environment }} - DEPLOY_AUTH: ${{ github.event.inputs.DEPLOY_AUTH}} DEPLOY_BACKEND: ${{ github.event.inputs.DEPLOY_BACKEND }} DEPLOY_FEATURES_API: ${{ github.event.inputs.DEPLOY_FEATURES_API }} DEPLOY_ROUTES: ${{ github.event.inputs.DEPLOY_ROUTES }} diff --git a/.github/workflows/update_deployment_status.yml b/.github/workflows/update_deployment_status.yml index 1d14cdc..afd3b83 100644 --- a/.github/workflows/update_deployment_status.yml +++ b/.github/workflows/update_deployment_status.yml @@ -32,7 +32,6 @@ jobs: python scripts/update_deployment_status.py \ --env "${{ inputs.environment }}" env: - VEDA_AUTH_GIT_REF: ${{ vars.VEDA_AUTH_GIT_REF }} VEDA_BACKEND_GIT_REF: ${{ vars.VEDA_BACKEND_GIT_REF }} VEDA_FEATURES_API_GIT_REF: ${{ vars.VEDA_FEATURES_API_GIT_REF }} VEDA_ROUTES_GIT_REF: ${{ vars.VEDA_ROUTES_GIT_REF }} @@ -42,7 +41,6 @@ jobs: VEDA_S3_DISASTER_RECOVERY_GIT_REF: ${{ vars.VEDA_S3_DISASTER_RECOVERY_GIT_REF }} VEDA_TITILER_CMR_GIT_REF: ${{ vars.VEDA_TITILER_CMR_GIT_REF }} DEPLOY_BACKEND: ${{ github.event.inputs.DEPLOY_BACKEND }} - DEPLOY_AUTH: ${{ github.event.inputs.DEPLOY_AUTH }} DEPLOY_FEATURES_API: ${{ github.event.inputs.DEPLOY_FEATURES_API }} DEPLOY_ROUTES: ${{ github.event.inputs.DEPLOY_ROUTES }} DEPLOY_SM2A: ${{ github.event.inputs.DEPLOY_SM2A }} diff --git a/.gitmodules b/.gitmodules index 6fd90a8..40be165 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,6 @@ [submodule "veda-data-airflow"] path = veda-data-airflow url = https://github.com/NASA-IMPACT/veda-data-airflow.git -[submodule "veda-auth"] - path = veda-auth - url = https://github.com/NASA-IMPACT/veda-auth.git [submodule "veda-backend"] path = veda-backend url = https://github.com/NASA-IMPACT/veda-backend.git diff --git a/README.md b/README.md index 9a59ff4..8ebb984 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,6 @@ More instructions on these Github environment variables is provided below. Git Ref for each project to use to deploy. Can be branch name, release tag or commit hash. Anything that works with `git checkout`. Below are some examples of the components that may be configured in a GitHub Environment. ```bash -VEDA_AUTH_GIT_REF= VEDA_BACKEND_GIT_REF= VEDA_FEATURES_API_GIT_REF= VEDA_SM2A_DATA_AIRFLOW_GIT_REF= @@ -113,7 +112,6 @@ To keep the components modular, each action should include all necessary steps f > Most deployments require [custom environment configuration](#store-env-configuration-in-aws-secrets-manager) that can be retrieved from the AWS Secrets Manager for the deployment. See [veda-backend/scripts/get-env.sh](https://github.com/NASA-IMPACT/veda-backend/blob/develop/scripts/get-env.sh) for an example environment configuration utility. ### Examples -- Veda-auth [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-auth/blob/main/.github/actions/cdk-deploy/action.yml) provides a simple example of adding configuration from an AWS Secrets Manager secret and running `cdk deploy` for an imported submodule. - Veda-backend [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/actions/cdk-deploy/action.yml) contains logic to run tests before deploying components. - This [CICD workflow in veda-backend](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/workflows/cicd.yml) demonstrates importing the cdk-deploy/action on a merge event to test the deployment in a dev enviornment. diff --git a/scripts/update_deployment_status.py b/scripts/update_deployment_status.py index f76272b..3c6e754 100644 --- a/scripts/update_deployment_status.py +++ b/scripts/update_deployment_status.py @@ -8,7 +8,6 @@ # component_id, DEPLOY_* env, VEDA_*_GIT_REF env, display name in MD, repo URL COMPONENTS = [ - ("auth", "DEPLOY_AUTH", "VEDA_AUTH_GIT_REF", "AUTH", "https://github.com/NASA-IMPACT/veda-auth"), ("backend", "DEPLOY_BACKEND", "VEDA_BACKEND_GIT_REF", "BACKEND", "https://github.com/NASA-IMPACT/veda-backend"), ("features_api", "DEPLOY_FEATURES_API", "VEDA_FEATURES_API_GIT_REF", "FEATURES_API", "https://github.com/NASA-IMPACT/veda-features-api-cdk"), ("routes", "DEPLOY_ROUTES", "VEDA_ROUTES_GIT_REF", "ROUTES", "https://github.com/NASA-IMPACT/veda-routes"), From 6176adb6a123602d34527d409727834b0ebc4ed1 Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Mon, 11 May 2026 13:59:12 -0700 Subject: [PATCH 205/210] chore: update required approvers list --- .github/workflows/dispatch.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 8f476b3..fd74594 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -59,14 +59,14 @@ on: description: DEPLOY_TITILER_CMR run-name: | - Dispatch to ${{ inputs.environment }} - AUTH=${{ inputs.DEPLOY_AUTH }} - BACKEND=${{ inputs.DEPLOY_BACKEND }} - FEATURES=${{ inputs.DEPLOY_FEATURES_API }} - ROUTES=${{ inputs.DEPLOY_ROUTES }} - SM2A=${{ inputs.DEPLOY_SM2A }} - MONITORING=${{ inputs.DEPLOY_MONITORING }} - TITILER_MULTIDIM=${{ inputs.DEPLOY_TITILER_MULTIDIM }} + Dispatch to ${{ inputs.environment }} + AUTH=${{ inputs.DEPLOY_AUTH }} + BACKEND=${{ inputs.DEPLOY_BACKEND }} + FEATURES=${{ inputs.DEPLOY_FEATURES_API }} + ROUTES=${{ inputs.DEPLOY_ROUTES }} + SM2A=${{ inputs.DEPLOY_SM2A }} + MONITORING=${{ inputs.DEPLOY_MONITORING }} + TITILER_MULTIDIM=${{ inputs.DEPLOY_TITILER_MULTIDIM }} TITILER_CMR=${{ inputs.DEPLOY_TITILER_CMR }} β›Ÿ env: @@ -93,7 +93,7 @@ jobs: # for the approval cannot exceed 60 minutes or the job will fail due to bad credentials with: secret: ${{ secrets.GITHUB_TOKEN }} - approvers: amarouane-ABDELHAK,slesaad,anayeaye,smohiudd,botanical,ividito,stephenkilbourn + approvers: amarouane-ABDELHAK,slesaad,anayeaye,smohiudd,botanical,ividito,stephenkilbourn,sandrahoang686,aliziel minimum-approvals: 1 issue-title: "Deploying to ${{ github.event.inputs.environment }}" issue-body: "Please approve or deny the deployment" From 128f22dc391204489051f5ae4a9852b953ea3364 Mon Sep 17 00:00:00 2001 From: Sandra Hoang Date: Wed, 13 May 2026 12:57:13 -0400 Subject: [PATCH 206/210] add back example submodule for cdk deploy --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8ebb984..4f37556 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ To keep the components modular, each action should include all necessary steps f > Most deployments require [custom environment configuration](#store-env-configuration-in-aws-secrets-manager) that can be retrieved from the AWS Secrets Manager for the deployment. See [veda-backend/scripts/get-env.sh](https://github.com/NASA-IMPACT/veda-backend/blob/develop/scripts/get-env.sh) for an example environment configuration utility. ### Examples +- Veda-features-api-cdk [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-features-api-cdk/blob/main/.github/actions/cdk-deploy/action.yml) provides a simple example of adding configuration from an AWS Secrets Manager secret and running `cdk deploy` for an imported submodule. - Veda-backend [cdk-deploy/action.yml](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/actions/cdk-deploy/action.yml) contains logic to run tests before deploying components. - This [CICD workflow in veda-backend](https://github.com/NASA-IMPACT/veda-backend/blob/develop/.github/workflows/cicd.yml) demonstrates importing the cdk-deploy/action on a merge event to test the deployment in a dev enviornment. From 32be24ebbc49ff2654415bef26c5a83d92dcafca Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Thu, 14 May 2026 12:50:51 -0700 Subject: [PATCH 207/210] fix: remove orphaned veda auth submodule --- veda-auth | 1 - 1 file changed, 1 deletion(-) delete mode 160000 veda-auth diff --git a/veda-auth b/veda-auth deleted file mode 160000 index ff1bb97..0000000 --- a/veda-auth +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ff1bb97dac80adac112d2264c34f7d9eead76729 From fdd0490f69169cf0731392e8abc04ac4d2676338 Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Thu, 4 Jun 2026 13:46:50 -0700 Subject: [PATCH 208/210] fix: update integration test env boolean parsing --- integration_test/test_api_health.py | 36 ++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/integration_test/test_api_health.py b/integration_test/test_api_health.py index 9530ba7..fbb04b7 100644 --- a/integration_test/test_api_health.py +++ b/integration_test/test_api_health.py @@ -1,20 +1,34 @@ +""" +API health checks for deployed VEDA backends +""" +import os + import requests from dotenv import load_dotenv -import os load_dotenv() + +def _env_bool(name: str, default: bool = False) -> bool: + """Parse a boolean env var. Treats false/0/ and empty string as False""" + raw = os.getenv(name) + if raw is None: + return default + return raw.strip().lower() in ("1", "true", "yes", "on") + + def _get_link(obj: dict, rel: str) -> str: """get rel link from a stac object""" - return next((l for l in obj.get("links") if l["rel"]==rel), None) + return next((l for l in obj.get("links") if l["rel"] == rel), None) + def test_stac_url_returns_200(): base_url = os.getenv("VEDA_STAC_URL") stac_root_path = os.getenv("VEDA_STAC_ROOT_PATH") custom_host = os.getenv("VEDA_CUSTOM_HOST", None) - disable_default_apigw = os.getenv("VEDA_DISABLE_DEFAULT_APIGW_ENDPOINT", False) + disable_default_apigw = _env_bool("VEDA_DISABLE_DEFAULT_APIGW_ENDPOINT") health_endpoint = "_mgmt/ping" - + if not disable_default_apigw: url = f"{base_url}{health_endpoint}" # APIGW base url includes trailing / print(f"Checking APIGW stac-api {url=}") @@ -26,14 +40,13 @@ def test_stac_url_returns_200(): print(f"Checking custom host stac-api {url=}") response = requests.get(url) assert response.status_code == 200 - def test_raster_url_returns_200(): base_url = os.getenv("VEDA_RASTER_URL") raster_root_path = os.getenv("VEDA_RASTER_ROOT_PATH") custom_host = os.getenv("VEDA_CUSTOM_HOST", None) - disable_default_apigw = os.getenv("VEDA_DISABLE_DEFAULT_APIGW_ENDPOINT", False) + disable_default_apigw = _env_bool("VEDA_DISABLE_DEFAULT_APIGW_ENDPOINT") health_endpoint = "healthz" if not disable_default_apigw: @@ -48,13 +61,14 @@ def test_raster_url_returns_200(): response = requests.get(url) assert response.status_code == 200 + def test_stac_item_next_link_returns_200(): base_url = os.getenv("VEDA_STAC_URL") stac_root_path = os.getenv("VEDA_STAC_ROOT_PATH") custom_host = os.getenv("VEDA_CUSTOM_HOST", None) - disable_default_apigw = os.getenv("VEDA_DISABLE_DEFAULT_APIGW_ENDPOINT", False) + disable_default_apigw = _env_bool("VEDA_DISABLE_DEFAULT_APIGW_ENDPOINT") collections_endpoint = "collections" - + if not disable_default_apigw: url = f"{base_url}/{collections_endpoint}" print(f"Checking APIGW stac-api {url=}") @@ -66,14 +80,14 @@ def test_stac_item_next_link_returns_200(): print(f"Checking links for custom host stac-api {url=}") response = requests.get(url) assert response.status_code == 200 - + # Walk check root path propagation through dynamic links when using custom host collections = response.json().get("collections") next_links_untested = True while next_links_untested: for collection in collections: - + # All collections should have a dynamicaly generated items link, even if no items exist items_link = _get_link(collection, "items") assert items_link @@ -83,7 +97,7 @@ def test_stac_item_next_link_returns_200(): assert items_response.status_code == 200 items_json = items_response.json() features = items_json.get("features") - + # The default page size is 10 if len(features) >= 10: items_next_link = _get_link(items_json, "next") From 0946e0dabcd45df43a9cec409b8f65b72179d4aa Mon Sep 17 00:00:00 2001 From: Jennifer Tran <12633533+botanical@users.noreply.github.com> Date: Thu, 4 Jun 2026 14:35:30 -0700 Subject: [PATCH 209/210] fix: simplify _env_bool function --- integration_test/test_api_health.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/integration_test/test_api_health.py b/integration_test/test_api_health.py index fbb04b7..f8d57a0 100644 --- a/integration_test/test_api_health.py +++ b/integration_test/test_api_health.py @@ -9,12 +9,11 @@ load_dotenv() -def _env_bool(name: str, default: bool = False) -> bool: - """Parse a boolean env var. Treats false/0/ and empty string as False""" +def _env_bool(name: str) -> bool: raw = os.getenv(name) if raw is None: - return default - return raw.strip().lower() in ("1", "true", "yes", "on") + return False + return raw.strip().lower() in ("1", "true") def _get_link(obj: dict, rel: str) -> str: From ded1669d6ef4192887ca3263750ac8386dc63d32 Mon Sep 17 00:00:00 2001 From: Henry Rodman Date: Wed, 10 Jun 2026 12:21:37 -0500 Subject: [PATCH 210/210] fix: remove Stephen from the list of approvers (#197) --- .github/workflows/dispatch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index ab329db..ce777cc 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -86,7 +86,7 @@ jobs: # for the approval cannot exceed 60 minutes or the job will fail due to bad credentials with: secret: ${{ secrets.GITHUB_TOKEN }} - approvers: amarouane-ABDELHAK,slesaad,anayeaye,smohiudd,botanical,ividito,stephenkilbourn,sandrahoang686,aliziel + approvers: amarouane-ABDELHAK,slesaad,anayeaye,smohiudd,botanical,ividito,sandrahoang686,aliziel minimum-approvals: 1 issue-title: "Deploying to ${{ github.event.inputs.environment }}" issue-body: "Please approve or deny the deployment"