From b30bc2d8be69237145a38871ab888583054e1761 Mon Sep 17 00:00:00 2001 From: Mangat Toor Date: Mon, 2 Jun 2025 18:09:15 -0700 Subject: [PATCH 01/27] ci: setup terraform script for provisioning dev environment --- terraform/dev/main.tf | 118 +++++++++++++++++++++++++++++++++++++ terraform/dev/variables.tf | 0 2 files changed, 118 insertions(+) create mode 100644 terraform/dev/main.tf create mode 100644 terraform/dev/variables.tf diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf new file mode 100644 index 0000000..21481de --- /dev/null +++ b/terraform/dev/main.tf @@ -0,0 +1,118 @@ +terraform { + backend "s3" { + bucket = "" + key = """ + region = "" + } +} + +provider "aws" { + region = "us-east-1" +} + +# Generate a new SSH key pair +resource "tls_private_key" "ec2_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +# Create AWS key pair from generated key +resource "aws_key_pair" "deployer" { + key_name = "generated-key" + public_key = tls_private_key.ec2_key.public_key_openssh +} + +# Security Group for RDS +resource "aws_security_group" "db_sg" { + name = "db-sg" + description = "Security group for RDS PostgreSQL instance" + + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + security_groups = [aws_security_group.app_sg.id] + description = "PostgreSQL access from EC2 app server" + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + description = "Allow all outbound traffic" + } +} + +# RDS DB +resource "aws_db_instance" "mydb" { + identifier = "myapp-db" + engine = "postgres" + instance_class = "db.t3.micro" + allocated_storage = 20 + db_name = var.db_name + username = var.db_user + password = var.db_password + skip_final_snapshot = true + publicly_accessible = false + # Security groups + vpc_security_group_ids = [aws_security_group.db_sg.id] + +} + +# Security Group for EC2 +resource "aws_security_group" "app_sg" { + name = "app-sg" + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] # SSH + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +# EC2 Instance +resource "aws_instance" "app" { + ami = "ami-0f88e80871fd81e91" + instance_type = "t2.micro" + key_name = aws_key_pair.deployer.key_name + vpc_security_group_ids = [aws_security_group.app_sg.id] + + tags = { + Name = "myapp-ec2" + } + + user_data = <<-EOF + #!/bin/bash + yum update -y + yum install -y docker git + service docker start + usermod -a -G docker ec2-user + chkconfig docker on + dnf install postgresql15 -y + sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + EOF +} + +# Outputs +output "ec2_ip" { + value = aws_instance.app.public_ip +} + +output "rds_endpoint" { + value = aws_db_instance.mydb.address +} + +output "private_key_pem" { + value = tls_private_key.ec2_key.private_key_pem + sensitive = true +} \ No newline at end of file diff --git a/terraform/dev/variables.tf b/terraform/dev/variables.tf new file mode 100644 index 0000000..e69de29 From ee63c05bb943ef463cd98833d194df41e6fe1d28 Mon Sep 17 00:00:00 2001 From: Mangat Toor Date: Mon, 2 Jun 2025 18:12:28 -0700 Subject: [PATCH 02/27] ci: setup github workflow for terraform provisioning and deployment --- .github/workflows/pipeline.yml | 251 +++++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 .github/workflows/pipeline.yml diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml new file mode 100644 index 0000000..4f9a9f8 --- /dev/null +++ b/.github/workflows/pipeline.yml @@ -0,0 +1,251 @@ +name: CI/CD Pipeline + +concurrency: + group: ci-${{ github.workflow }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: + - dev + +permissions: + contents: read + actions: write + +jobs: + validate-secrets: + runs-on: ubuntu-latest + steps: + - name: Validate required secrets + run: | + REQUIRED_SECRETS=( + "AWS_ACCESS_KEY_ID" + "AWS_SECRET_ACCESS_KEY" + "BUCKET_NAME" + "BUCKET_KEY" + "DB_PASSWORD" + "DJANGO_SECRET_KEY" + "DJANGO_DEBUG" + "AIRBNB_PUBLIC_API_KEY" + "POSTGRES_USER" + "POSTGRES_DB" + "POSTGRES_HOST_PORT" + "CELERY_BROKER_URL" + "CELERY_RESULT_BACKEND" + "GH_TOKEN" + ) + + MISSING=false + + for secret in "${REQUIRED_SECRETS[@]}"; do + if [ -z "${{ secrets[secret] }}" ]; then + echo "❌ Missing secret: $secret" + MISSING=true + fi + done + + if [ "$MISSING" = true ]; then + echo "❌ One or more required secrets are missing. Failing workflow." + exit 1 + else + echo "✅ All required secrets are set." + fi + + # Check if Terraform files have changed + check-changes: + needs: validate-secrets + runs-on: ubuntu-latest + outputs: + terraform-changed: ${{ steps.terraform-changes.outputs.changed }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check for changes in Terraform directory + id: terraform-changes + run: | + echo "Checking for changes in terraform/dev..." + if ! git diff --name-only ${{ github.sha }} | grep -q '^terraform/dev/'; then + echo "Terraform files changed." + echo "changed=true" >> $GITHUB_OUTPUT + else + echo "No changes in terraform/dev." + echo "changed=false" >> $GITHUB_OUTPUT + fi + # Provision infrastructure (only when terraform changes) + provision: + runs-on: ubuntu-latest + needs: check-changes + steps: + - name: Checkout + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: actions/checkout@v4 + + - name: Configure AWS credentials + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Setup Terraform + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init and Apply + if: needs.check-changes.outputs.terraform-changed == 'true' + env: + TF_VAR_db_password: ${{ secrets.DB_PASSWORD }} + run: | + cd terraform/dev + terraform init -backend-config="bucket=${{secrets.BUCKET_NAME}}" -backend-config="key=${{secrets.BUCKET_KEY}}" -backend-config="region=us-east-1" + terraform apply -auto-approve + + - name: Save Terraform outputs + if: needs.check-changes.outputs.terraform-changed == 'true' + working-directory: terraform/dev + run: | + terraform output -json > tf_outputs.json + cat tf_outputs.json + + - name: Save Private Key + if: needs.check-changes.outputs.terraform-changed == 'true' + working-directory: terraform/dev + run: | + terraform output -raw private_key_pem > private_key.pem + + - name: Upload secrets + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: hkusu/s3-upload-action@v2 + id: upload + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: 'us-east-1' + aws-bucket: ${{ secrets.BUCKET_NAME }} + # TODO: Need to change this variables are dynamic + destination-dir: 'terraform/dev/' + file-path: './terraform/dev/tf_outputs.json' + # TODO: Maybe should be false I think + output-file-url: 'true' + + # Deploy application (always runs, but waits for provision if it ran) + deploy: + runs-on: ubuntu-latest + needs: [check-changes, provision] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Print start message + run: | + echo "Starting EC2 deployment workflow..." + echo "Terraform changed: ${{ needs.check-changes.outputs.terraform-changed }}" + echo "Provision job result: ${{ needs.provision.result }}" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + + - name: Download from S3 + # TODO: Again path should be dynamic or in a variable + run: | + aws s3 cp s3://${{ secrets.BUCKET_NAME }}/artifacts/terraform/dev/tf_outputs.json ./tf_outputs.json + echo "✅ Downloaded tf_outputs.json" + ls -la tf_outputs.json + + - name: Display tf_outputs.json content + run: cat ./tf_outputs.json + + - name: Parse Terraform outputs + id: tf + run: | + ec2_ip=$(jq -r '.ec2_ip.value' ./tf_outputs.json) + rds_url=$(jq -r '.rds_endpoint.value' ./tf_outputs.json) + echo "Parsed EC2 IP: $ec2_ip" + echo "Parsed RDS URL: $rds_url" + echo "ec2_ip=$ec2_ip" >> $GITHUB_OUTPUT + echo "rds_url=$rds_url" >> $GITHUB_OUTPUT + + - name: Read private key into environment variable + id: read_key + run: | + echo "Reading private key into environment variable..." + if jq -e '.private_key_pem.value' ./tf_outputs.json > /dev/null; then + echo "PRIVATE_KEY<> $GITHUB_ENV + jq -r '.private_key_pem.value' ./tf_outputs.json >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "✅ Private key successfully added to environment variable." + else + echo "❌ Failed to parse private key from tf_outputs.json" + exit 1 + fi + + - name: Deploy to EC2 instance + uses: appleboy/ssh-action@v1.2.0 + with: + host: ${{ steps.tf.outputs.ec2_ip }} + username: ec2-user + key: ${{ env.PRIVATE_KEY }} + # TODO: Move the variables into a env file + script: | + echo "Connected to EC2 instance at ${{ steps.tf.outputs.ec2_ip }}" + git config --global credential.helper store + echo "https://username:${{ secrets.GH_TOKEN }}@github.com" > ~/.git-credentials + cd /home/ec2-user/ + echo "Checking if project directory exists..." + if [ ! -d "airbnb-regulation" ]; then + echo "Cloning project repository..." + git clone https://github.com/CodeForBc/airbnb-regulation + fi + + cd airbnb-regulation + echo "Pulling latest changes from main branch..." + git pull origin main + + echo "Creating .env file with environment variables..." + cat < .env + SECRET_KEY="${{ secrets.DJANGO_SECRET_KEY }}" + DJANGO_DEBUG=${{ secrets.DJANGO_DEBUG }} + AIRBNB_PUBLIC_API_KEY="${{ secrets.AIRBNB_PUBLIC_API_KEY }}" + POSTGRES_PASSWORD=${{ secrets.DB_PASSWORD }} + POSTGRES_USER=${{ secrets.POSTGRES_USER }} + POSTGRES_URL=${{ steps.tf.outputs.rds_url }} + POSTGRES_DB=${{ secrets.POSTGRES_DB }} + POSTGRES_HOST_PORT=${{ secrets.POSTGRES_HOST_PORT }} + CELERY_BROKER_URL=${{ secrets.CELERY_BROKER_URL }} + CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND }} + EOF + echo ".env file created." + + if docker ps --format '{{.Names}}' | grep -q '^airbnb_celery$'; then + while true; do + result=$(docker exec airbnb_celery celery -A airbnb_project inspect active 2>&1) + echo "Celery active task output: $result" + + if [[ "$result" == *"empty"* ]]; then + echo "No active Celery tasks detected." + break + fi + + echo "Active Celery tasks found. Waiting..." + sleep 10 + done + else + echo "Celery container 'airbnb_celery' is not running. Skipping task check." + fi + + echo "Stopping Docker containers..." + docker-compose down + + echo "Rebuilding and starting Docker containers..." + docker-compose up -d --build + + echo "Deployment process completed." \ No newline at end of file From 58279cd83a6b0663e313a1c2393fef37ae193841 Mon Sep 17 00:00:00 2001 From: Mangat Toor Date: Sat, 7 Jun 2025 10:39:05 -0700 Subject: [PATCH 03/27] chore: increase retry to give more time for db to come online --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d17f409..bd06ac9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] interval: 30s timeout: 10s - retries: 5 + retries: 15 policies: container_name: airbnb_policies From 2de220e89430807ba4c9351bd4fd3824878b20b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sat, 7 Jun 2025 16:49:19 -0700 Subject: [PATCH 04/27] feat: add variables to terraform --- terraform/dev/main.tf | 8 ++++---- terraform/dev/variables.tf | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf index 21481de..6bb51e5 100644 --- a/terraform/dev/main.tf +++ b/terraform/dev/main.tf @@ -1,7 +1,7 @@ terraform { backend "s3" { bucket = "" - key = """ + key = "" region = "" } } @@ -50,9 +50,9 @@ resource "aws_db_instance" "mydb" { engine = "postgres" instance_class = "db.t3.micro" allocated_storage = 20 - db_name = var.db_name - username = var.db_user - password = var.db_password + db_name = var.db_name_dev + username = var.db_user_dev + password = var.db_password_dev skip_final_snapshot = true publicly_accessible = false # Security groups diff --git a/terraform/dev/variables.tf b/terraform/dev/variables.tf index e69de29..b5e5f97 100644 --- a/terraform/dev/variables.tf +++ b/terraform/dev/variables.tf @@ -0,0 +1,7 @@ +variable "bucket_name_dev" {default = ""} +variable "bucket_key_dev" {default = ""} +variable "key_name_dev" {default = ""} +variable "public_key_path_dev" {default = ""} +variable "db_name_dev" { default = "" } +variable "db_user_dev" { default = "" } +variable "db_password_dev" {default = ""} From acad88c692e4db10d2fbba31852cb8f7b8b7b999 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sat, 7 Jun 2025 16:50:13 -0700 Subject: [PATCH 05/27] feat: update github secrets to use dev secrets --- .github/workflows/pipeline.yml | 72 +++++++++++++++++----------------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 4f9a9f8..ee544be 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -21,20 +21,20 @@ jobs: - name: Validate required secrets run: | REQUIRED_SECRETS=( - "AWS_ACCESS_KEY_ID" - "AWS_SECRET_ACCESS_KEY" - "BUCKET_NAME" - "BUCKET_KEY" - "DB_PASSWORD" - "DJANGO_SECRET_KEY" - "DJANGO_DEBUG" - "AIRBNB_PUBLIC_API_KEY" - "POSTGRES_USER" - "POSTGRES_DB" - "POSTGRES_HOST_PORT" - "CELERY_BROKER_URL" - "CELERY_RESULT_BACKEND" - "GH_TOKEN" + "AWS_ACCESS_KEY_ID_DEV" + "AWS_SECRET_ACCESS_KEY_DEV" + "BUCKET_NAME_DEV" + "BUCKET_KEY_DEV" + "DB_PASSWORD_DEV" + "DJANGO_SECRET_KEY_DEV" + "DJANGO_DEBUG_DEV" + "AIRBNB_PUBLIC_API_KEY_DEV" + "POSTGRES_USER_DEV" + "POSTGRES_DB_DEV" + "POSTGRES_HOST_PORT_DEV" + "CELERY_BROKER_URL_DEV" + "CELERY_RESULT_BACKEND_DEV" + "GH_TOKEN_DEV" ) MISSING=false @@ -87,8 +87,8 @@ jobs: if: needs.check-changes.outputs.terraform-changed == 'true' uses: aws-actions/configure-aws-credentials@v3 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEV }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} aws-region: us-east-1 - name: Setup Terraform @@ -98,10 +98,10 @@ jobs: - name: Terraform Init and Apply if: needs.check-changes.outputs.terraform-changed == 'true' env: - TF_VAR_db_password: ${{ secrets.DB_PASSWORD }} + TF_VAR_db_password: ${{ secrets.DB_PASSWORD_DEV }} run: | cd terraform/dev - terraform init -backend-config="bucket=${{secrets.BUCKET_NAME}}" -backend-config="key=${{secrets.BUCKET_KEY}}" -backend-config="region=us-east-1" + terraform init -backend-config="bucket=${{secrets.BUCKET_NAME_DEV}}" -backend-config="key=${{secrets.BUCKET_KEY_DEV}}" -backend-config="region=us-east-1" terraform apply -auto-approve - name: Save Terraform outputs @@ -122,13 +122,13 @@ jobs: uses: hkusu/s3-upload-action@v2 id: upload with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEV }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} aws-region: 'us-east-1' aws-bucket: ${{ secrets.BUCKET_NAME }} # TODO: Need to change this variables are dynamic - destination-dir: 'terraform/dev/' - file-path: './terraform/dev/tf_outputs.json' + destination-dir: '' + file-path: 'tf_outputs.json' # TODO: Maybe should be false I think output-file-url: 'true' @@ -150,14 +150,14 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEV }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} aws-region: us-east-1 - name: Download from S3 # TODO: Again path should be dynamic or in a variable run: | - aws s3 cp s3://${{ secrets.BUCKET_NAME }}/artifacts/terraform/dev/tf_outputs.json ./tf_outputs.json + aws s3 cp s3://${{ secrets.BUCKET_NAME_DEV}}/tf_outputs.json ./tf_outputs.json echo "✅ Downloaded tf_outputs.json" ls -la tf_outputs.json @@ -197,8 +197,6 @@ jobs: # TODO: Move the variables into a env file script: | echo "Connected to EC2 instance at ${{ steps.tf.outputs.ec2_ip }}" - git config --global credential.helper store - echo "https://username:${{ secrets.GH_TOKEN }}@github.com" > ~/.git-credentials cd /home/ec2-user/ echo "Checking if project directory exists..." if [ ! -d "airbnb-regulation" ]; then @@ -212,16 +210,16 @@ jobs: echo "Creating .env file with environment variables..." cat < .env - SECRET_KEY="${{ secrets.DJANGO_SECRET_KEY }}" - DJANGO_DEBUG=${{ secrets.DJANGO_DEBUG }} - AIRBNB_PUBLIC_API_KEY="${{ secrets.AIRBNB_PUBLIC_API_KEY }}" - POSTGRES_PASSWORD=${{ secrets.DB_PASSWORD }} - POSTGRES_USER=${{ secrets.POSTGRES_USER }} - POSTGRES_URL=${{ steps.tf.outputs.rds_url }} - POSTGRES_DB=${{ secrets.POSTGRES_DB }} - POSTGRES_HOST_PORT=${{ secrets.POSTGRES_HOST_PORT }} - CELERY_BROKER_URL=${{ secrets.CELERY_BROKER_URL }} - CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND }} + SECRET_KEY="${{ secrets.DJANGO_SECRET_KEY_DEV }}" + DJANGO_DEBUG=${{ secrets.DJANGO_DEBUG_DEV }} + AIRBNB_PUBLIC_API_KEY="${{ secrets.AIRBNB_PUBLIC_API_KEY_DEV }}" + POSTGRES_PASSWORD=${{ secrets.DB_PASSWORD_DEV }} + POSTGRES_USER=${{ secrets.POSTGRES_USER_DEV }} + POSTGRES_URL=${{ steps.tf.outputs.rds_url_DEV }} + POSTGRES_DB=${{ secrets.POSTGRES_DB_DEV }} + POSTGRES_HOST_PORT=${{ secrets.POSTGRES_HOST_PORT_DEV }} + CELERY_BROKER_URL=${{ secrets.CELERY_BROKER_URL_DEV }} + CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND_DEV }} EOF echo ".env file created." From cd5a72204116315b153ced59f411ec81c695f5f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sat, 7 Jun 2025 16:54:46 -0700 Subject: [PATCH 06/27] ci: trigger workflow on pr creation for testing --- .github/workflows/pipeline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index ee544be..f16f03b 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -5,6 +5,7 @@ concurrency: cancel-in-progress: true on: + pull_reqeust: workflow_dispatch: push: branches: @@ -135,7 +136,7 @@ jobs: # Deploy application (always runs, but waits for provision if it ran) deploy: runs-on: ubuntu-latest - needs: [check-changes, provision] + needs: [ check-changes, provision ] steps: - name: Checkout repository From 174381bfe12e89b575a8bf68fa8355aae262f559 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sat, 7 Jun 2025 16:56:14 -0700 Subject: [PATCH 07/27] ci: fix typo --- .github/workflows/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index f16f03b..9c1dabc 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -5,7 +5,7 @@ concurrency: cancel-in-progress: true on: - pull_reqeust: + pull_request: workflow_dispatch: push: branches: From 6ddd99a6dddb9d95a7ac99aadab6f1010e9e6917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sat, 7 Jun 2025 16:59:02 -0700 Subject: [PATCH 08/27] ci: fix check secrets --- .github/workflows/pipeline.yml | 72 ++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 9c1dabc..d7dce82 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -18,42 +18,46 @@ permissions: jobs: validate-secrets: runs-on: ubuntu-latest - steps: - - name: Validate required secrets - run: | - REQUIRED_SECRETS=( - "AWS_ACCESS_KEY_ID_DEV" - "AWS_SECRET_ACCESS_KEY_DEV" - "BUCKET_NAME_DEV" - "BUCKET_KEY_DEV" - "DB_PASSWORD_DEV" - "DJANGO_SECRET_KEY_DEV" - "DJANGO_DEBUG_DEV" - "AIRBNB_PUBLIC_API_KEY_DEV" - "POSTGRES_USER_DEV" - "POSTGRES_DB_DEV" - "POSTGRES_HOST_PORT_DEV" - "CELERY_BROKER_URL_DEV" - "CELERY_RESULT_BACKEND_DEV" - "GH_TOKEN_DEV" - ) - - MISSING=false - - for secret in "${REQUIRED_SECRETS[@]}"; do - if [ -z "${{ secrets[secret] }}" ]; then - echo "❌ Missing secret: $secret" - MISSING=true - fi - done - - if [ "$MISSING" = true ]; then - echo "❌ One or more required secrets are missing. Failing workflow." - exit 1 + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + run: | + echo "🔍 Validating required secrets..." + + # Required secrets list + REQUIRED_SECRETS=( + "AWS_ACCESS_KEY_ID_DEV" + "AWS_SECRET_ACCESS_KEY_DEV" + "BUCKET_NAME_DEV" + "BUCKET_KEY_DEV" + "DB_PASSWORD_DEV" + "DJANGO_SECRET_KEY_DEV" + "DJANGO_DEBUG_DEV" + "AIRBNB_PUBLIC_API_KEY_DEV" + "POSTGRES_USER_DEV" + "POSTGRES_DB_DEV" + "POSTGRES_HOST_PORT_DEV" + "CELERY_BROKER_URL_DEV" + "CELERY_RESULT_BACKEND_DEV" + "GH_TOKEN_DEV" + ) + + MISSING=false + + for secret in "${REQUIRED_SECRETS[@]}"; do + if ! echo "$SECRETS_CONTEXT" | jq -e --arg key "$secret" 'has($key)' >/dev/null; then + echo "❌ Missing secret: $secret" + MISSING=true else - echo "✅ All required secrets are set." + echo "✅ Found secret: $secret" fi - + done + + if [ "$MISSING" = true ]; then + echo "❌ One or more required secrets are missing. Failing workflow." + exit 1 + else + echo "✅ All required secrets are set." + fi # Check if Terraform files have changed check-changes: needs: validate-secrets From 3144529b1e4d3b37c42711f47206082c7a24086d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sat, 7 Jun 2025 17:02:32 -0700 Subject: [PATCH 09/27] ci: fix check secrets --- .github/workflows/pipeline.yml | 74 +++++++++++++++++----------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index d7dce82..ab946f2 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -20,44 +20,46 @@ jobs: runs-on: ubuntu-latest env: SECRETS_CONTEXT: ${{ toJson(secrets) }} - run: | - echo "🔍 Validating required secrets..." - - # Required secrets list - REQUIRED_SECRETS=( - "AWS_ACCESS_KEY_ID_DEV" - "AWS_SECRET_ACCESS_KEY_DEV" - "BUCKET_NAME_DEV" - "BUCKET_KEY_DEV" - "DB_PASSWORD_DEV" - "DJANGO_SECRET_KEY_DEV" - "DJANGO_DEBUG_DEV" - "AIRBNB_PUBLIC_API_KEY_DEV" - "POSTGRES_USER_DEV" - "POSTGRES_DB_DEV" - "POSTGRES_HOST_PORT_DEV" - "CELERY_BROKER_URL_DEV" - "CELERY_RESULT_BACKEND_DEV" - "GH_TOKEN_DEV" - ) - - MISSING=false - - for secret in "${REQUIRED_SECRETS[@]}"; do - if ! echo "$SECRETS_CONTEXT" | jq -e --arg key "$secret" 'has($key)' >/dev/null; then - echo "❌ Missing secret: $secret" - MISSING=true + steps: + - name: Validate required secrets + run: | + echo "🔍 Validating required secrets..." + + # Required secrets list + REQUIRED_SECRETS=( + "AWS_ACCESS_KEY_ID_DEV" + "AWS_SECRET_ACCESS_KEY_DEV" + "BUCKET_NAME_DEV" + "BUCKET_KEY_DEV" + "DB_PASSWORD_DEV" + "DJANGO_SECRET_KEY_DEV" + "DJANGO_DEBUG_DEV" + "AIRBNB_PUBLIC_API_KEY_DEV" + "POSTGRES_USER_DEV" + "POSTGRES_DB_DEV" + "POSTGRES_HOST_PORT_DEV" + "CELERY_BROKER_URL_DEV" + "CELERY_RESULT_BACKEND_DEV" + "GH_TOKEN_DEV" + ) + + MISSING=false + + for secret in "${REQUIRED_SECRETS[@]}"; do + if ! echo "$SECRETS_CONTEXT" | jq -e --arg key "$secret" 'has($key)' >/dev/null; then + echo "❌ Missing secret: $secret" + MISSING=true + else + echo "✅ Found secret: $secret" + fi + done + + if [ "$MISSING" = true ]; then + echo "❌ One or more required secrets are missing. Failing workflow." + exit 1 else - echo "✅ Found secret: $secret" + echo "✅ All required secrets are set." fi - done - - if [ "$MISSING" = true ]; then - echo "❌ One or more required secrets are missing. Failing workflow." - exit 1 - else - echo "✅ All required secrets are set." - fi # Check if Terraform files have changed check-changes: needs: validate-secrets From 365a2e0fd180f1513522c1a90afe3eef9b2d0069 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sat, 7 Jun 2025 17:03:02 -0700 Subject: [PATCH 10/27] ci: fix check secrets --- .github/workflows/pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index ab946f2..87f7cce 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -40,7 +40,6 @@ jobs: "POSTGRES_HOST_PORT_DEV" "CELERY_BROKER_URL_DEV" "CELERY_RESULT_BACKEND_DEV" - "GH_TOKEN_DEV" ) MISSING=false From 658de90eba3d869b7ab78987ddd63dd7fd3f3ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sat, 7 Jun 2025 17:08:51 -0700 Subject: [PATCH 11/27] ci: fix check pipeline --- .github/workflows/pipeline.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 87f7cce..96e8948 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -108,7 +108,10 @@ jobs: run: | cd terraform/dev terraform init -backend-config="bucket=${{secrets.BUCKET_NAME_DEV}}" -backend-config="key=${{secrets.BUCKET_KEY_DEV}}" -backend-config="region=us-east-1" - terraform apply -auto-approve + terraform apply -auto-approve \ + -var="db_name=${{ secrets.POSTGRES_DB_DEV }}" \ + -var="username=${{ secrets.POSTGRES_USER_DEV }}" \ + -var="password=${{ secrets.DB_PASSWORD_DEV }}" - name: Save Terraform outputs if: needs.check-changes.outputs.terraform-changed == 'true' From cc3d92b5461988473a9a473b14385b5bd291e37b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sat, 7 Jun 2025 17:10:20 -0700 Subject: [PATCH 12/27] ci: fix check pipeline --- .github/workflows/pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 96e8948..ed97cd5 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -109,9 +109,9 @@ jobs: cd terraform/dev terraform init -backend-config="bucket=${{secrets.BUCKET_NAME_DEV}}" -backend-config="key=${{secrets.BUCKET_KEY_DEV}}" -backend-config="region=us-east-1" terraform apply -auto-approve \ - -var="db_name=${{ secrets.POSTGRES_DB_DEV }}" \ - -var="username=${{ secrets.POSTGRES_USER_DEV }}" \ - -var="password=${{ secrets.DB_PASSWORD_DEV }}" + -var="db_name_dev=${{ secrets.POSTGRES_DB_DEV }}" \ + -var="username_dev=${{ secrets.POSTGRES_USER_DEV }}" \ + -var="password_dev=${{ secrets.DB_PASSWORD_DEV }}" - name: Save Terraform outputs if: needs.check-changes.outputs.terraform-changed == 'true' From d2815e72a3fee7806be26b8e6d80b2bfe5f893e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sat, 7 Jun 2025 17:11:39 -0700 Subject: [PATCH 13/27] ci: fix check pipeline --- .github/workflows/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index ed97cd5..915dfc1 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -110,8 +110,8 @@ jobs: terraform init -backend-config="bucket=${{secrets.BUCKET_NAME_DEV}}" -backend-config="key=${{secrets.BUCKET_KEY_DEV}}" -backend-config="region=us-east-1" terraform apply -auto-approve \ -var="db_name_dev=${{ secrets.POSTGRES_DB_DEV }}" \ - -var="username_dev=${{ secrets.POSTGRES_USER_DEV }}" \ - -var="password_dev=${{ secrets.DB_PASSWORD_DEV }}" + -var="db_username_dev=${{ secrets.POSTGRES_USER_DEV }}" \ + -var="db_password_dev=${{ secrets.DB_PASSWORD_DEV }}" - name: Save Terraform outputs if: needs.check-changes.outputs.terraform-changed == 'true' From 161f94645c0154e06cb43d07a5b42fd4ec5115da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sat, 7 Jun 2025 17:13:05 -0700 Subject: [PATCH 14/27] ci: fix check pipeline --- .github/workflows/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 915dfc1..0b32690 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -110,7 +110,7 @@ jobs: terraform init -backend-config="bucket=${{secrets.BUCKET_NAME_DEV}}" -backend-config="key=${{secrets.BUCKET_KEY_DEV}}" -backend-config="region=us-east-1" terraform apply -auto-approve \ -var="db_name_dev=${{ secrets.POSTGRES_DB_DEV }}" \ - -var="db_username_dev=${{ secrets.POSTGRES_USER_DEV }}" \ + -var="db_user_dev=${{ secrets.POSTGRES_USER_DEV }}" \ -var="db_password_dev=${{ secrets.DB_PASSWORD_DEV }}" - name: Save Terraform outputs From 6fb43cb376eebfae924a521c548100498216eb4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sun, 8 Jun 2025 14:23:58 -0700 Subject: [PATCH 15/27] ci: fix check pipeline --- .github/workflows/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 0b32690..6317017 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -134,7 +134,7 @@ jobs: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEV }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} aws-region: 'us-east-1' - aws-bucket: ${{ secrets.BUCKET_NAME }} + aws-bucket: ${{ secrets.BUCKET_NAME_DEV }} # TODO: Need to change this variables are dynamic destination-dir: '' file-path: 'tf_outputs.json' From d660e5df857f0fdc0cdc7e120840222213a07581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sun, 8 Jun 2025 14:26:03 -0700 Subject: [PATCH 16/27] ci: fix check pipeline --- .github/workflows/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 6317017..d062c54 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -137,9 +137,9 @@ jobs: aws-bucket: ${{ secrets.BUCKET_NAME_DEV }} # TODO: Need to change this variables are dynamic destination-dir: '' - file-path: 'tf_outputs.json' + file-path: './tf_outputs.json' # TODO: Maybe should be false I think - output-file-url: 'true' + output-file-url: 'false' # Deploy application (always runs, but waits for provision if it ran) deploy: From 6167c66b58bb33986351019ee2290722bde4960f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sun, 8 Jun 2025 14:28:23 -0700 Subject: [PATCH 17/27] ci: fix check pipeline --- .github/workflows/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index d062c54..b718c49 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -137,7 +137,7 @@ jobs: aws-bucket: ${{ secrets.BUCKET_NAME_DEV }} # TODO: Need to change this variables are dynamic destination-dir: '' - file-path: './tf_outputs.json' + file-path: './terraform/dev/tf_outputs.json' # TODO: Maybe should be false I think output-file-url: 'false' From 789224fdf2e4f7609368efa68f948a38a5c14570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sun, 8 Jun 2025 14:31:17 -0700 Subject: [PATCH 18/27] ci: fix check pipeline --- .github/workflows/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index b718c49..d8ee459 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -136,7 +136,7 @@ jobs: aws-region: 'us-east-1' aws-bucket: ${{ secrets.BUCKET_NAME_DEV }} # TODO: Need to change this variables are dynamic - destination-dir: '' + destination-dir: '/' file-path: './terraform/dev/tf_outputs.json' # TODO: Maybe should be false I think output-file-url: 'false' From 310378ba9c03ec54bf84dae7930faf76a64f2ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sun, 8 Jun 2025 14:33:53 -0700 Subject: [PATCH 19/27] ci: fix check pipeline --- .github/workflows/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index d8ee459..020395d 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -135,11 +135,11 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} aws-region: 'us-east-1' aws-bucket: ${{ secrets.BUCKET_NAME_DEV }} + bucket-root: '/' # TODO: Need to change this variables are dynamic destination-dir: '/' file-path: './terraform/dev/tf_outputs.json' # TODO: Maybe should be false I think - output-file-url: 'false' # Deploy application (always runs, but waits for provision if it ran) deploy: From 3a1fe8d5aae6a7dbf3a8dffa67257192528b945d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sun, 8 Jun 2025 14:39:21 -0700 Subject: [PATCH 20/27] ci: fix check pipeline --- .github/workflows/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 020395d..88311ab 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -224,7 +224,7 @@ jobs: AIRBNB_PUBLIC_API_KEY="${{ secrets.AIRBNB_PUBLIC_API_KEY_DEV }}" POSTGRES_PASSWORD=${{ secrets.DB_PASSWORD_DEV }} POSTGRES_USER=${{ secrets.POSTGRES_USER_DEV }} - POSTGRES_URL=${{ steps.tf.outputs.rds_url_DEV }} + POSTGRES_URL=${{ steps.tf.outputs.rds_url }} POSTGRES_DB=${{ secrets.POSTGRES_DB_DEV }} POSTGRES_HOST_PORT=${{ secrets.POSTGRES_HOST_PORT_DEV }} CELERY_BROKER_URL=${{ secrets.CELERY_BROKER_URL_DEV }} From efd48a44329557f95a22c58a6cc6fd56db6c45f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sun, 8 Jun 2025 14:50:24 -0700 Subject: [PATCH 21/27] ci: fix check pipeline --- .github/workflows/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 88311ab..4a946d5 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -224,7 +224,7 @@ jobs: AIRBNB_PUBLIC_API_KEY="${{ secrets.AIRBNB_PUBLIC_API_KEY_DEV }}" POSTGRES_PASSWORD=${{ secrets.DB_PASSWORD_DEV }} POSTGRES_USER=${{ secrets.POSTGRES_USER_DEV }} - POSTGRES_URL=${{ steps.tf.outputs.rds_url }} + POSTGRES_URL=${{ steps.tf.outputs.rds_url}} POSTGRES_DB=${{ secrets.POSTGRES_DB_DEV }} POSTGRES_HOST_PORT=${{ secrets.POSTGRES_HOST_PORT_DEV }} CELERY_BROKER_URL=${{ secrets.CELERY_BROKER_URL_DEV }} From 9773c891c002ae6f7afc23340efdb01d83c18f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sun, 8 Jun 2025 15:12:02 -0700 Subject: [PATCH 22/27] feat: add script to setup cron job --- .github/workflows/pipeline.yml | 5 ++++- scripts/setup_cron.sh | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 scripts/setup_cron.sh diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 4a946d5..fa28485 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -254,5 +254,8 @@ jobs: echo "Rebuilding and starting Docker containers..." docker-compose up -d --build - + + echo "Setting up cron jobs" + chmod +x scripts/setup_cron.sh + ./scripts/setup_cron.sh echo "Deployment process completed." \ No newline at end of file diff --git a/scripts/setup_cron.sh b/scripts/setup_cron.sh new file mode 100644 index 0000000..7da5108 --- /dev/null +++ b/scripts/setup_cron.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +CRON_JOB="0 0 * * * curl -s http://localhost:8001/listings/harvest-listings/ > /dev/null 2>&1" + +# Check if the cron job already exists +crontab -l 2>/dev/null | grep -F "$CRON_JOB" >/dev/null + +if [ $? -eq 0 ]; then + echo "Cron job already exists." +else + (crontab -l 2>/dev/null; echo "$CRON_JOB") | crontab - + echo "Cron job added successfully." +fi From f0270dcf794f094432d82effec20c3b58073e118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sun, 8 Jun 2025 15:19:28 -0700 Subject: [PATCH 23/27] feat: add script to set up cron job --- .github/workflows/pipeline.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index fa28485..28d6e2c 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -255,7 +255,8 @@ jobs: echo "Rebuilding and starting Docker containers..." docker-compose up -d --build - echo "Setting up cron jobs" - chmod +x scripts/setup_cron.sh - ./scripts/setup_cron.sh - echo "Deployment process completed." \ No newline at end of file + echo "Setting up cron jobs..." + cd /home/ec2-user/airbnb-regulation # make sure you're in the right directory + chmod +x scripts/setup_cron.sh + ./scripts/setup_cron.sh + echo "✅ Deployment process completed." \ No newline at end of file From 5b249c84d139dab8c5f9e661f129618ad78267e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sun, 8 Jun 2025 15:20:17 -0700 Subject: [PATCH 24/27] feat: add script to set up cron job --- .github/workflows/pipeline.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 28d6e2c..9050d11 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -255,8 +255,8 @@ jobs: echo "Rebuilding and starting Docker containers..." docker-compose up -d --build - echo "Setting up cron jobs..." - cd /home/ec2-user/airbnb-regulation # make sure you're in the right directory - chmod +x scripts/setup_cron.sh - ./scripts/setup_cron.sh - echo "✅ Deployment process completed." \ No newline at end of file + echo "Setting up cron jobs..." + cd /home/ec2-user/airbnb-regulation # make sure you're in the right directory + chmod +x scripts/setup_cron.sh + ./scripts/setup_cron.sh + echo "✅ Deployment process completed." \ No newline at end of file From d9e807103acdb01a873ef17b12de3b40ac7ad5d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sun, 8 Jun 2025 15:24:38 -0700 Subject: [PATCH 25/27] feat: add script to set up cron job --- .github/workflows/pipeline.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index 9050d11..a97b538 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -215,7 +215,7 @@ jobs: cd airbnb-regulation echo "Pulling latest changes from main branch..." - git pull origin main + git pull origin dev echo "Creating .env file with environment variables..." cat < .env @@ -256,7 +256,6 @@ jobs: docker-compose up -d --build echo "Setting up cron jobs..." - cd /home/ec2-user/airbnb-regulation # make sure you're in the right directory chmod +x scripts/setup_cron.sh ./scripts/setup_cron.sh echo "✅ Deployment process completed." \ No newline at end of file From 2738bdf75d2c1d83e7f9222d7bd64bbb5c7f06ad Mon Sep 17 00:00:00 2001 From: Mangat Toor Date: Mon, 2 Jun 2025 18:09:15 -0700 Subject: [PATCH 26/27] ci: Implement CI/CD pipeline with Terraform Adds a GitHub Actions workflow to automate the deployment process. - Provisions infrastructure on AWS using Terraform. - Deploys the application to an EC2 instance. - Includes secret validation and a cron job setup script. --- .github/workflows/pipeline.yml | 261 +++++++++++++++++++++++++++++++++ docker-compose.yml | 2 +- scripts/setup_cron.sh | 13 ++ terraform/dev/main.tf | 118 +++++++++++++++ terraform/dev/variables.tf | 7 + 5 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pipeline.yml create mode 100644 scripts/setup_cron.sh create mode 100644 terraform/dev/main.tf create mode 100644 terraform/dev/variables.tf diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml new file mode 100644 index 0000000..a97b538 --- /dev/null +++ b/.github/workflows/pipeline.yml @@ -0,0 +1,261 @@ +name: CI/CD Pipeline + +concurrency: + group: ci-${{ github.workflow }} + cancel-in-progress: true + +on: + pull_request: + workflow_dispatch: + push: + branches: + - dev + +permissions: + contents: read + actions: write + +jobs: + validate-secrets: + runs-on: ubuntu-latest + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + steps: + - name: Validate required secrets + run: | + echo "🔍 Validating required secrets..." + + # Required secrets list + REQUIRED_SECRETS=( + "AWS_ACCESS_KEY_ID_DEV" + "AWS_SECRET_ACCESS_KEY_DEV" + "BUCKET_NAME_DEV" + "BUCKET_KEY_DEV" + "DB_PASSWORD_DEV" + "DJANGO_SECRET_KEY_DEV" + "DJANGO_DEBUG_DEV" + "AIRBNB_PUBLIC_API_KEY_DEV" + "POSTGRES_USER_DEV" + "POSTGRES_DB_DEV" + "POSTGRES_HOST_PORT_DEV" + "CELERY_BROKER_URL_DEV" + "CELERY_RESULT_BACKEND_DEV" + ) + + MISSING=false + + for secret in "${REQUIRED_SECRETS[@]}"; do + if ! echo "$SECRETS_CONTEXT" | jq -e --arg key "$secret" 'has($key)' >/dev/null; then + echo "❌ Missing secret: $secret" + MISSING=true + else + echo "✅ Found secret: $secret" + fi + done + + if [ "$MISSING" = true ]; then + echo "❌ One or more required secrets are missing. Failing workflow." + exit 1 + else + echo "✅ All required secrets are set." + fi + # Check if Terraform files have changed + check-changes: + needs: validate-secrets + runs-on: ubuntu-latest + outputs: + terraform-changed: ${{ steps.terraform-changes.outputs.changed }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check for changes in Terraform directory + id: terraform-changes + run: | + echo "Checking for changes in terraform/dev..." + if ! git diff --name-only ${{ github.sha }} | grep -q '^terraform/dev/'; then + echo "Terraform files changed." + echo "changed=true" >> $GITHUB_OUTPUT + else + echo "No changes in terraform/dev." + echo "changed=false" >> $GITHUB_OUTPUT + fi + # Provision infrastructure (only when terraform changes) + provision: + runs-on: ubuntu-latest + needs: check-changes + steps: + - name: Checkout + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: actions/checkout@v4 + + - name: Configure AWS credentials + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEV }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} + aws-region: us-east-1 + + - name: Setup Terraform + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init and Apply + if: needs.check-changes.outputs.terraform-changed == 'true' + env: + TF_VAR_db_password: ${{ secrets.DB_PASSWORD_DEV }} + run: | + cd terraform/dev + terraform init -backend-config="bucket=${{secrets.BUCKET_NAME_DEV}}" -backend-config="key=${{secrets.BUCKET_KEY_DEV}}" -backend-config="region=us-east-1" + terraform apply -auto-approve \ + -var="db_name_dev=${{ secrets.POSTGRES_DB_DEV }}" \ + -var="db_user_dev=${{ secrets.POSTGRES_USER_DEV }}" \ + -var="db_password_dev=${{ secrets.DB_PASSWORD_DEV }}" + + - name: Save Terraform outputs + if: needs.check-changes.outputs.terraform-changed == 'true' + working-directory: terraform/dev + run: | + terraform output -json > tf_outputs.json + cat tf_outputs.json + + - name: Save Private Key + if: needs.check-changes.outputs.terraform-changed == 'true' + working-directory: terraform/dev + run: | + terraform output -raw private_key_pem > private_key.pem + + - name: Upload secrets + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: hkusu/s3-upload-action@v2 + id: upload + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEV }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} + aws-region: 'us-east-1' + aws-bucket: ${{ secrets.BUCKET_NAME_DEV }} + bucket-root: '/' + # TODO: Need to change this variables are dynamic + destination-dir: '/' + file-path: './terraform/dev/tf_outputs.json' + # TODO: Maybe should be false I think + + # Deploy application (always runs, but waits for provision if it ran) + deploy: + runs-on: ubuntu-latest + needs: [ check-changes, provision ] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Print start message + run: | + echo "Starting EC2 deployment workflow..." + echo "Terraform changed: ${{ needs.check-changes.outputs.terraform-changed }}" + echo "Provision job result: ${{ needs.provision.result }}" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEV }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} + aws-region: us-east-1 + + - name: Download from S3 + # TODO: Again path should be dynamic or in a variable + run: | + aws s3 cp s3://${{ secrets.BUCKET_NAME_DEV}}/tf_outputs.json ./tf_outputs.json + echo "✅ Downloaded tf_outputs.json" + ls -la tf_outputs.json + + - name: Display tf_outputs.json content + run: cat ./tf_outputs.json + + - name: Parse Terraform outputs + id: tf + run: | + ec2_ip=$(jq -r '.ec2_ip.value' ./tf_outputs.json) + rds_url=$(jq -r '.rds_endpoint.value' ./tf_outputs.json) + echo "Parsed EC2 IP: $ec2_ip" + echo "Parsed RDS URL: $rds_url" + echo "ec2_ip=$ec2_ip" >> $GITHUB_OUTPUT + echo "rds_url=$rds_url" >> $GITHUB_OUTPUT + + - name: Read private key into environment variable + id: read_key + run: | + echo "Reading private key into environment variable..." + if jq -e '.private_key_pem.value' ./tf_outputs.json > /dev/null; then + echo "PRIVATE_KEY<> $GITHUB_ENV + jq -r '.private_key_pem.value' ./tf_outputs.json >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "✅ Private key successfully added to environment variable." + else + echo "❌ Failed to parse private key from tf_outputs.json" + exit 1 + fi + + - name: Deploy to EC2 instance + uses: appleboy/ssh-action@v1.2.0 + with: + host: ${{ steps.tf.outputs.ec2_ip }} + username: ec2-user + key: ${{ env.PRIVATE_KEY }} + # TODO: Move the variables into a env file + script: | + echo "Connected to EC2 instance at ${{ steps.tf.outputs.ec2_ip }}" + cd /home/ec2-user/ + echo "Checking if project directory exists..." + if [ ! -d "airbnb-regulation" ]; then + echo "Cloning project repository..." + git clone https://github.com/CodeForBc/airbnb-regulation + fi + + cd airbnb-regulation + echo "Pulling latest changes from main branch..." + git pull origin dev + + echo "Creating .env file with environment variables..." + cat < .env + SECRET_KEY="${{ secrets.DJANGO_SECRET_KEY_DEV }}" + DJANGO_DEBUG=${{ secrets.DJANGO_DEBUG_DEV }} + AIRBNB_PUBLIC_API_KEY="${{ secrets.AIRBNB_PUBLIC_API_KEY_DEV }}" + POSTGRES_PASSWORD=${{ secrets.DB_PASSWORD_DEV }} + POSTGRES_USER=${{ secrets.POSTGRES_USER_DEV }} + POSTGRES_URL=${{ steps.tf.outputs.rds_url}} + POSTGRES_DB=${{ secrets.POSTGRES_DB_DEV }} + POSTGRES_HOST_PORT=${{ secrets.POSTGRES_HOST_PORT_DEV }} + CELERY_BROKER_URL=${{ secrets.CELERY_BROKER_URL_DEV }} + CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND_DEV }} + EOF + echo ".env file created." + + if docker ps --format '{{.Names}}' | grep -q '^airbnb_celery$'; then + while true; do + result=$(docker exec airbnb_celery celery -A airbnb_project inspect active 2>&1) + echo "Celery active task output: $result" + + if [[ "$result" == *"empty"* ]]; then + echo "No active Celery tasks detected." + break + fi + + echo "Active Celery tasks found. Waiting..." + sleep 10 + done + else + echo "Celery container 'airbnb_celery' is not running. Skipping task check." + fi + + echo "Stopping Docker containers..." + docker-compose down + + echo "Rebuilding and starting Docker containers..." + docker-compose up -d --build + + echo "Setting up cron jobs..." + chmod +x scripts/setup_cron.sh + ./scripts/setup_cron.sh + echo "✅ Deployment process completed." \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d17f409..bd06ac9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] interval: 30s timeout: 10s - retries: 5 + retries: 15 policies: container_name: airbnb_policies diff --git a/scripts/setup_cron.sh b/scripts/setup_cron.sh new file mode 100644 index 0000000..7da5108 --- /dev/null +++ b/scripts/setup_cron.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +CRON_JOB="0 0 * * * curl -s http://localhost:8001/listings/harvest-listings/ > /dev/null 2>&1" + +# Check if the cron job already exists +crontab -l 2>/dev/null | grep -F "$CRON_JOB" >/dev/null + +if [ $? -eq 0 ]; then + echo "Cron job already exists." +else + (crontab -l 2>/dev/null; echo "$CRON_JOB") | crontab - + echo "Cron job added successfully." +fi diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf new file mode 100644 index 0000000..6bb51e5 --- /dev/null +++ b/terraform/dev/main.tf @@ -0,0 +1,118 @@ +terraform { + backend "s3" { + bucket = "" + key = "" + region = "" + } +} + +provider "aws" { + region = "us-east-1" +} + +# Generate a new SSH key pair +resource "tls_private_key" "ec2_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +# Create AWS key pair from generated key +resource "aws_key_pair" "deployer" { + key_name = "generated-key" + public_key = tls_private_key.ec2_key.public_key_openssh +} + +# Security Group for RDS +resource "aws_security_group" "db_sg" { + name = "db-sg" + description = "Security group for RDS PostgreSQL instance" + + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + security_groups = [aws_security_group.app_sg.id] + description = "PostgreSQL access from EC2 app server" + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + description = "Allow all outbound traffic" + } +} + +# RDS DB +resource "aws_db_instance" "mydb" { + identifier = "myapp-db" + engine = "postgres" + instance_class = "db.t3.micro" + allocated_storage = 20 + db_name = var.db_name_dev + username = var.db_user_dev + password = var.db_password_dev + skip_final_snapshot = true + publicly_accessible = false + # Security groups + vpc_security_group_ids = [aws_security_group.db_sg.id] + +} + +# Security Group for EC2 +resource "aws_security_group" "app_sg" { + name = "app-sg" + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] # SSH + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +# EC2 Instance +resource "aws_instance" "app" { + ami = "ami-0f88e80871fd81e91" + instance_type = "t2.micro" + key_name = aws_key_pair.deployer.key_name + vpc_security_group_ids = [aws_security_group.app_sg.id] + + tags = { + Name = "myapp-ec2" + } + + user_data = <<-EOF + #!/bin/bash + yum update -y + yum install -y docker git + service docker start + usermod -a -G docker ec2-user + chkconfig docker on + dnf install postgresql15 -y + sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + EOF +} + +# Outputs +output "ec2_ip" { + value = aws_instance.app.public_ip +} + +output "rds_endpoint" { + value = aws_db_instance.mydb.address +} + +output "private_key_pem" { + value = tls_private_key.ec2_key.private_key_pem + sensitive = true +} \ No newline at end of file diff --git a/terraform/dev/variables.tf b/terraform/dev/variables.tf new file mode 100644 index 0000000..b5e5f97 --- /dev/null +++ b/terraform/dev/variables.tf @@ -0,0 +1,7 @@ +variable "bucket_name_dev" {default = ""} +variable "bucket_key_dev" {default = ""} +variable "key_name_dev" {default = ""} +variable "public_key_path_dev" {default = ""} +variable "db_name_dev" { default = "" } +variable "db_user_dev" { default = "" } +variable "db_password_dev" {default = ""} From 7102976d91a68bff5c3948b948443b4b4f9010a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mangat=20Singh=20Toor=20=7C=20=E0=A8=AE=E0=A9=B0=E0=A8=97?= =?UTF-8?q?=E0=A8=A4=20=E0=A8=B8=E0=A8=BF=E0=A9=B0=E0=A8=98=20=E0=A8=A4?= =?UTF-8?q?=E0=A9=82=E0=A8=B0?= Date: Sun, 8 Jun 2025 15:34:30 -0700 Subject: [PATCH 27/27] chore: remove pull request trigger from workflow --- .github/workflows/pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml index a97b538..fe9d2be 100644 --- a/.github/workflows/pipeline.yml +++ b/.github/workflows/pipeline.yml @@ -5,7 +5,6 @@ concurrency: cancel-in-progress: true on: - pull_request: workflow_dispatch: push: branches: