diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml new file mode 100644 index 0000000..fe9d2be --- /dev/null +++ b/.github/workflows/pipeline.yml @@ -0,0 +1,260 @@ +name: CI/CD Pipeline + +concurrency: + group: ci-${{ github.workflow }} + cancel-in-progress: true + +on: + workflow_dispatch: + push: + branches: + - dev + +permissions: + contents: read + actions: write + +jobs: + validate-secrets: + runs-on: ubuntu-latest + env: + SECRETS_CONTEXT: ${{ toJson(secrets) }} + steps: + - name: Validate required secrets + run: | + echo "🔍 Validating required secrets..." + + # Required secrets list + REQUIRED_SECRETS=( + "AWS_ACCESS_KEY_ID_DEV" + "AWS_SECRET_ACCESS_KEY_DEV" + "BUCKET_NAME_DEV" + "BUCKET_KEY_DEV" + "DB_PASSWORD_DEV" + "DJANGO_SECRET_KEY_DEV" + "DJANGO_DEBUG_DEV" + "AIRBNB_PUBLIC_API_KEY_DEV" + "POSTGRES_USER_DEV" + "POSTGRES_DB_DEV" + "POSTGRES_HOST_PORT_DEV" + "CELERY_BROKER_URL_DEV" + "CELERY_RESULT_BACKEND_DEV" + ) + + MISSING=false + + for secret in "${REQUIRED_SECRETS[@]}"; do + if ! echo "$SECRETS_CONTEXT" | jq -e --arg key "$secret" 'has($key)' >/dev/null; then + echo "❌ Missing secret: $secret" + MISSING=true + else + echo "✅ Found secret: $secret" + fi + done + + if [ "$MISSING" = true ]; then + echo "❌ One or more required secrets are missing. Failing workflow." + exit 1 + else + echo "✅ All required secrets are set." + fi + # Check if Terraform files have changed + check-changes: + needs: validate-secrets + runs-on: ubuntu-latest + outputs: + terraform-changed: ${{ steps.terraform-changes.outputs.changed }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check for changes in Terraform directory + id: terraform-changes + run: | + echo "Checking for changes in terraform/dev..." + if ! git diff --name-only ${{ github.sha }} | grep -q '^terraform/dev/'; then + echo "Terraform files changed." + echo "changed=true" >> $GITHUB_OUTPUT + else + echo "No changes in terraform/dev." + echo "changed=false" >> $GITHUB_OUTPUT + fi + # Provision infrastructure (only when terraform changes) + provision: + runs-on: ubuntu-latest + needs: check-changes + steps: + - name: Checkout + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: actions/checkout@v4 + + - name: Configure AWS credentials + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: aws-actions/configure-aws-credentials@v3 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEV }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} + aws-region: us-east-1 + + - name: Setup Terraform + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init and Apply + if: needs.check-changes.outputs.terraform-changed == 'true' + env: + TF_VAR_db_password: ${{ secrets.DB_PASSWORD_DEV }} + run: | + cd terraform/dev + terraform init -backend-config="bucket=${{secrets.BUCKET_NAME_DEV}}" -backend-config="key=${{secrets.BUCKET_KEY_DEV}}" -backend-config="region=us-east-1" + terraform apply -auto-approve \ + -var="db_name_dev=${{ secrets.POSTGRES_DB_DEV }}" \ + -var="db_user_dev=${{ secrets.POSTGRES_USER_DEV }}" \ + -var="db_password_dev=${{ secrets.DB_PASSWORD_DEV }}" + + - name: Save Terraform outputs + if: needs.check-changes.outputs.terraform-changed == 'true' + working-directory: terraform/dev + run: | + terraform output -json > tf_outputs.json + cat tf_outputs.json + + - name: Save Private Key + if: needs.check-changes.outputs.terraform-changed == 'true' + working-directory: terraform/dev + run: | + terraform output -raw private_key_pem > private_key.pem + + - name: Upload secrets + if: needs.check-changes.outputs.terraform-changed == 'true' + uses: hkusu/s3-upload-action@v2 + id: upload + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEV }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} + aws-region: 'us-east-1' + aws-bucket: ${{ secrets.BUCKET_NAME_DEV }} + bucket-root: '/' + # TODO: Need to change this variables are dynamic + destination-dir: '/' + file-path: './terraform/dev/tf_outputs.json' + # TODO: Maybe should be false I think + + # Deploy application (always runs, but waits for provision if it ran) + deploy: + runs-on: ubuntu-latest + needs: [ check-changes, provision ] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Print start message + run: | + echo "Starting EC2 deployment workflow..." + echo "Terraform changed: ${{ needs.check-changes.outputs.terraform-changed }}" + echo "Provision job result: ${{ needs.provision.result }}" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEV }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEV }} + aws-region: us-east-1 + + - name: Download from S3 + # TODO: Again path should be dynamic or in a variable + run: | + aws s3 cp s3://${{ secrets.BUCKET_NAME_DEV}}/tf_outputs.json ./tf_outputs.json + echo "✅ Downloaded tf_outputs.json" + ls -la tf_outputs.json + + - name: Display tf_outputs.json content + run: cat ./tf_outputs.json + + - name: Parse Terraform outputs + id: tf + run: | + ec2_ip=$(jq -r '.ec2_ip.value' ./tf_outputs.json) + rds_url=$(jq -r '.rds_endpoint.value' ./tf_outputs.json) + echo "Parsed EC2 IP: $ec2_ip" + echo "Parsed RDS URL: $rds_url" + echo "ec2_ip=$ec2_ip" >> $GITHUB_OUTPUT + echo "rds_url=$rds_url" >> $GITHUB_OUTPUT + + - name: Read private key into environment variable + id: read_key + run: | + echo "Reading private key into environment variable..." + if jq -e '.private_key_pem.value' ./tf_outputs.json > /dev/null; then + echo "PRIVATE_KEY<> $GITHUB_ENV + jq -r '.private_key_pem.value' ./tf_outputs.json >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "✅ Private key successfully added to environment variable." + else + echo "❌ Failed to parse private key from tf_outputs.json" + exit 1 + fi + + - name: Deploy to EC2 instance + uses: appleboy/ssh-action@v1.2.0 + with: + host: ${{ steps.tf.outputs.ec2_ip }} + username: ec2-user + key: ${{ env.PRIVATE_KEY }} + # TODO: Move the variables into a env file + script: | + echo "Connected to EC2 instance at ${{ steps.tf.outputs.ec2_ip }}" + cd /home/ec2-user/ + echo "Checking if project directory exists..." + if [ ! -d "airbnb-regulation" ]; then + echo "Cloning project repository..." + git clone https://github.com/CodeForBc/airbnb-regulation + fi + + cd airbnb-regulation + echo "Pulling latest changes from main branch..." + git pull origin dev + + echo "Creating .env file with environment variables..." + cat < .env + SECRET_KEY="${{ secrets.DJANGO_SECRET_KEY_DEV }}" + DJANGO_DEBUG=${{ secrets.DJANGO_DEBUG_DEV }} + AIRBNB_PUBLIC_API_KEY="${{ secrets.AIRBNB_PUBLIC_API_KEY_DEV }}" + POSTGRES_PASSWORD=${{ secrets.DB_PASSWORD_DEV }} + POSTGRES_USER=${{ secrets.POSTGRES_USER_DEV }} + POSTGRES_URL=${{ steps.tf.outputs.rds_url}} + POSTGRES_DB=${{ secrets.POSTGRES_DB_DEV }} + POSTGRES_HOST_PORT=${{ secrets.POSTGRES_HOST_PORT_DEV }} + CELERY_BROKER_URL=${{ secrets.CELERY_BROKER_URL_DEV }} + CELERY_RESULT_BACKEND=${{ secrets.CELERY_RESULT_BACKEND_DEV }} + EOF + echo ".env file created." + + if docker ps --format '{{.Names}}' | grep -q '^airbnb_celery$'; then + while true; do + result=$(docker exec airbnb_celery celery -A airbnb_project inspect active 2>&1) + echo "Celery active task output: $result" + + if [[ "$result" == *"empty"* ]]; then + echo "No active Celery tasks detected." + break + fi + + echo "Active Celery tasks found. Waiting..." + sleep 10 + done + else + echo "Celery container 'airbnb_celery' is not running. Skipping task check." + fi + + echo "Stopping Docker containers..." + docker-compose down + + echo "Rebuilding and starting Docker containers..." + docker-compose up -d --build + + echo "Setting up cron jobs..." + chmod +x scripts/setup_cron.sh + ./scripts/setup_cron.sh + echo "✅ Deployment process completed." \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d17f409..bd06ac9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] interval: 30s timeout: 10s - retries: 5 + retries: 15 policies: container_name: airbnb_policies diff --git a/scripts/setup_cron.sh b/scripts/setup_cron.sh new file mode 100644 index 0000000..7da5108 --- /dev/null +++ b/scripts/setup_cron.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +CRON_JOB="0 0 * * * curl -s http://localhost:8001/listings/harvest-listings/ > /dev/null 2>&1" + +# Check if the cron job already exists +crontab -l 2>/dev/null | grep -F "$CRON_JOB" >/dev/null + +if [ $? -eq 0 ]; then + echo "Cron job already exists." +else + (crontab -l 2>/dev/null; echo "$CRON_JOB") | crontab - + echo "Cron job added successfully." +fi diff --git a/terraform/dev/main.tf b/terraform/dev/main.tf new file mode 100644 index 0000000..6bb51e5 --- /dev/null +++ b/terraform/dev/main.tf @@ -0,0 +1,118 @@ +terraform { + backend "s3" { + bucket = "" + key = "" + region = "" + } +} + +provider "aws" { + region = "us-east-1" +} + +# Generate a new SSH key pair +resource "tls_private_key" "ec2_key" { + algorithm = "RSA" + rsa_bits = 4096 +} + +# Create AWS key pair from generated key +resource "aws_key_pair" "deployer" { + key_name = "generated-key" + public_key = tls_private_key.ec2_key.public_key_openssh +} + +# Security Group for RDS +resource "aws_security_group" "db_sg" { + name = "db-sg" + description = "Security group for RDS PostgreSQL instance" + + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + security_groups = [aws_security_group.app_sg.id] + description = "PostgreSQL access from EC2 app server" + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + description = "Allow all outbound traffic" + } +} + +# RDS DB +resource "aws_db_instance" "mydb" { + identifier = "myapp-db" + engine = "postgres" + instance_class = "db.t3.micro" + allocated_storage = 20 + db_name = var.db_name_dev + username = var.db_user_dev + password = var.db_password_dev + skip_final_snapshot = true + publicly_accessible = false + # Security groups + vpc_security_group_ids = [aws_security_group.db_sg.id] + +} + +# Security Group for EC2 +resource "aws_security_group" "app_sg" { + name = "app-sg" + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] # SSH + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +# EC2 Instance +resource "aws_instance" "app" { + ami = "ami-0f88e80871fd81e91" + instance_type = "t2.micro" + key_name = aws_key_pair.deployer.key_name + vpc_security_group_ids = [aws_security_group.app_sg.id] + + tags = { + Name = "myapp-ec2" + } + + user_data = <<-EOF + #!/bin/bash + yum update -y + yum install -y docker git + service docker start + usermod -a -G docker ec2-user + chkconfig docker on + dnf install postgresql15 -y + sudo curl -L https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + EOF +} + +# Outputs +output "ec2_ip" { + value = aws_instance.app.public_ip +} + +output "rds_endpoint" { + value = aws_db_instance.mydb.address +} + +output "private_key_pem" { + value = tls_private_key.ec2_key.private_key_pem + sensitive = true +} \ No newline at end of file diff --git a/terraform/dev/variables.tf b/terraform/dev/variables.tf new file mode 100644 index 0000000..b5e5f97 --- /dev/null +++ b/terraform/dev/variables.tf @@ -0,0 +1,7 @@ +variable "bucket_name_dev" {default = ""} +variable "bucket_key_dev" {default = ""} +variable "key_name_dev" {default = ""} +variable "public_key_path_dev" {default = ""} +variable "db_name_dev" { default = "" } +variable "db_user_dev" { default = "" } +variable "db_password_dev" {default = ""}