diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..8d26bfc --- /dev/null +++ b/.env.sample @@ -0,0 +1,26 @@ +# vim: filetype=bash + +COMPOSE_PROFILES=core +COMPOSE_PROJECT_NAME=mapswipe-deploy +MAPSWIPE_ENVIRONMENT=prod|stage + +# Ofelia +OFELIA_PROJECT_NAME=mapswipe-prod +OFELIA_SLACK_WEBHOOK= + +# Postgres +# NOTE: Password: Use openssl rand -base64 48 | tr -dc 'A-Za-z0-9_@#%^+=-' | head -c 24 +POSTGRES_USER= +POSTGRES_PASSWORD= + +# Caddy +CADDY_EMAIL=mapswipe@example.org +CADDY_HOST_BACKEND=https://backend.example.org +CADDY_HOST_MANAGER_DASHBOARD=https://manager.example.org +CADDY_HOST_COMMUNITY_DASHBOARD=https://community.example.org + +# PgBackRest +# https://pgbackrest.org/command.html +# https://pgbackrest.org/configuration.html#introduction +PGBACKREST_REPO1_GCS_BUCKET=demo-bucket +PGBACKREST_REPO1_PATH=/demo-repo diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 0000000..ef5e746 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,47 @@ +name: Pre commit + +on: + push: + +jobs: + pre_commit_checks: + name: Pre-Commit checks + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@main + with: + submodules: true + + - name: Generate fake ./terraform/live/secrets.auto.tfvars + run: | + echo "project_id_map = { stage = \"mapswipe\" }" > ./terraform/live/secrets.auto.tfvars + + # TODO: Cache plugins? + - uses: terraform-linters/setup-tflint@v5 + name: Setup TFLint + with: + tflint_version: v0.52.0 + + - name: Install Terragrunt and OpenTofu + uses: gruntwork-io/terragrunt-action@v3 + with: + # TODO: Use mise instead? https://github.com/gruntwork-io/terragrunt-action#tool-version-management + tg_version: 0.80.4 + tofu_version: v1.10.5 + + - name: Initialize tofu without any backend + run: | + # Find all directories containing terragrunt.hcl + find . -type f -name "terragrunt.hcl" | while read hcl_file; do + dir=$(dirname "$hcl_file") + echo "Entering directory: $dir" + ( + cd "$dir" || exit + tofu init -backend=false + ) + done + + - uses: pre-commit/action@main + env: + DISABLE_INIT: true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..30bda03 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.env* +!.env.sample +data/ diff --git a/.gitmodules b/.gitmodules index 4898f16..69b120d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,8 +1,8 @@ [submodule "mapswipe-firebase"] - path = mapswipe-firebase + path = firebase url = git@github.com:mapswipe/mapswipe-firebase.git [submodule "mapswipe-backend"] - path = mapswipe-backend + path = backend url = git@github.com:mapswipe/mapswipe-backend.git [submodule "community-dashboard"] path = community-dashboard diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..7eae012 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,45 @@ +repos: + - repo: https://github.com/gitleaks/gitleaks + rev: v8.24.2 + hooks: + - id: gitleaks + + - repo: https://github.com/crate-ci/typos + rev: v1.31.1 + hooks: + - id: typos + args: ["--exclude=CHANGELOG.md", "--force-exclude"] + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-case-conflict + - id: detect-private-key + - id: check-merge-conflict + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-json + - id: check-toml + - id: check-xml + - id: check-yaml + args: [--unsafe] + - id: debug-statements + - id: detect-private-key + + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: "v1.100.0" + hooks: + - id: terraform_fmt + - id: terraform_tflint + - id: terragrunt_fmt + - id: terragrunt_validate_inputs + files: (/terragrunt\.hcl)$ + - id: terragrunt_validate + files: (/terragrunt\.hcl)$ + args: + - --hook-config=--tf-path=tofu + # - id: terraform_trivy + # - id: infracost_breakdown diff --git a/README.md b/README.md new file mode 100644 index 0000000..5f97492 --- /dev/null +++ b/README.md @@ -0,0 +1,68 @@ +## Setup + +### Cloning the Repository + +Most submodules use SSH URLs. To avoid setting up SSH keys, run this command to use HTTPS instead: + +```bash +git config --global url."https://github.com/".insteadOf "git@github.com:" +``` + +Clone and pull all submodules +```bash +git clone git@github.com:mapswipe/mapswipe-deploy.git +cd mapswipe-deploy +git submodule update --init --recursive +``` + +### Environment Variables + +Make sure these environment files are in place: + +- .env (based on `.env.sample`) +- env/backend.env +- env/community-dashboard.env +- env/manager-dashboard.env +- secrets/pgbackrest_gc_service_account_key.json + +```bash +cp .env.sample .env +touch env/backend.env +touch env/community-dashboard.env +touch env/manager-dashboard.env +touch secrets/pgbackrest_gc_service_account_key.json +``` + +## Apply changes + +The `task` tool is used to set up a pre-alias. +> https://taskfile.dev/ + + +```bash +task --list-all + +# Deploy all +task deploy + +# Deploy web apps +task web-builds + +# Deploy backend resources +task backend-deploy + +# Deploy caddy +task caddy-deploy +``` + +### pgBackRest + +Create "main" stanza +```bash +docker compose exec -u postgres postgres pgbackrest --stanza=main stanza-create +``` + +View backup info +```bash +docker compose exec -u postgres postgres pgbackrest --stanza=main info +``` diff --git a/backend b/backend new file mode 160000 index 0000000..126ed64 --- /dev/null +++ b/backend @@ -0,0 +1 @@ +Subproject commit 126ed6431298d79c90cd46b2fc25b745f1ebb12a diff --git a/caddy/Caddyfile b/caddy/Caddyfile new file mode 100644 index 0000000..8529c28 --- /dev/null +++ b/caddy/Caddyfile @@ -0,0 +1,43 @@ +(file_server_config) { + encode gzip + file_server +} + + +{$CADDY_HOST_BACKEND} { + request_body { + max_size 10MB + } + + handle_path /static/* { + root * /assets/backend/static + import file_server_config + } + + handle_path /media/* { + root * /assets/backend/media + import file_server_config + } + + handle { + reverse_proxy http://web:80 + } +} + + +{$CADDY_HOST_MANAGER_DASHBOARD} { + handle { + try_files {path} /index.html + root * /assets/manager-dashboard + import file_server_config + } +} + + +{$CADDY_HOST_COMMUNITY_DASHBOARD} { + handle { + try_files {path} /index.html + root * /assets/community-dashboard + import file_server_config + } +} diff --git a/community-dashboard b/community-dashboard index 9892645..ced6945 160000 --- a/community-dashboard +++ b/community-dashboard @@ -1 +1 @@ -Subproject commit 9892645ee15d24f155bf3810151e4d5491b4a375 +Subproject commit ced6945483d3dedeb580dcdde21ad34211daa24c diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..05adae4 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,245 @@ +name: ${COMPOSE_PROJECT_NAME?error} + +x-server: &base_server_setup + build: + context: ./backend/ + tags: + - mapswipe/mapswipe-backend:${BACKEND_DOCKER_TAG:-latest} + # Used for python debugging. + stdin_open: true + tty: true + restart: unless-stopped + environment: &base_server_environments + APP_RELEASE: ${BACKEND_COMMIT_HASH} + DEBUG: ${BACKEND_DEBUG:-false} + ENABLE_DEBUG_TOOLBAR: ${BACKEND_ENABLE_DEBUG_TOOLBAR:-false} + APP_ENVIRONMENT: ${MAPSWIPE_ENVIRONMENT:-PROD} + ENABLE_STRAWBERRY_GRAPHIQL: ${BACKEND_ENABLE_STRAWBERRY_GRAPHIQL:-false} + # Postgres + POSTGRES_HOST: ${POSTGRES_HOST:-postgres} + POSTGRES_PORT: ${POSTGRES_PORT:-5432} + POSTGRES_DB: ${POSTGRES_DB:-mapswipe} + POSTGRES_USER: ${POSTGRES_USER?error} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD?error} + # Redis (Dragonfly) + CELERY_REDIS_URL: ${BACKEND_CELERY_REDIS_URL:-redis://dragonfly:6379/0} + CACHE_REDIS_URL: ${BACKEND_CACHE_REDIS_URL:-redis://dragonfly:6379/1} + TEST_CACHE_REDIS_URL: ${BACKEND_TEST_CACHE_REDIS_URL:-redis://dragonfly:6379/11} + # Email (TODO: Dummy config) + EMAIL_HOST: ${BACKEND_EMAIL_HOST:-dummy} + EMAIL_PORT: ${BACKEND_EMAIL_PORT:-1025} + EMAIL_USE_TLS: ${BACKEND_EMAIL_USE_TLS:-false} + EMAIL_HOST_USER: ${BACKEND_EMAIL_HOST_USER:-dummy} + EMAIL_HOST_PASSWORD: ${BACKEND_EMAIL_HOST_PASSWORD:-dummy} + DEFAULT_FROM_EMAIL: ${BACKEND_DEFAULT_FROM_EMAIL:-Mapswipe Dev } + # Firebase + FIREBASE_EMULATOR_USE: false + # Storage + MEDIA_ROOT: "/data/media" + STATIC_ROOT: "/data/static" + MEDIA_URL: "media/" + STATIC_URL: "static/" + env_file: + - ./env/backend.env + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - ./data/backend/static:/data/static + - ./data/backend/media:/data/media + - ipython_data_local:/root/.ipython/profile_default # persist ipython data, including ipython history + depends_on: + - postgres + - dragonfly + profiles: + - core + + +x-worker: &base_worker_setup + <<: *base_server_setup + environment: + <<: *base_server_environments + APP_TYPE: "WORKER-BEAT" + +services: + ofelia: + image: mcuadros/ofelia:0.3.17 + command: daemon --docker -f label=com.docker.compose.project=${COMPOSE_PROJECT_NAME?error} + restart: unless-stopped + volumes: + - ./data/ofelia/:/logs + - /var/run/docker.sock:/var/run/docker.sock:ro + labels: + # File log + ofelia.save-folder: /logs + ofelia.save-only-on-error: 0 + # Slack + ofelia.slack-webhook: ${OFELIA_SLACK_WEBHOOK?error} + ofelia.slack-only-on-error: 1 + # Folder Cleanup + # Disk check (Every hour) + ofelia.job-local.cleanup.schedule: "@hourly" + ofelia.job-local.cleanup.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} find /logs/ -type f -mtime +7 -delete -print" + profiles: + - core + + # Backend + postgres: + build: + context: ./postgres/ + dockerfile: Dockerfile + target: postgis + restart: unless-stopped + ports: + - 127.0.0.1:${POSTGRES_DB_LOCAL_EXPOSE_PORT:-5432}:5432 + environment: + # NOTE: This config is only used for db first startup only + POSTGRES_DB: ${POSTGRES_DB:-mapswipe} + POSTGRES_USER: ${POSTGRES_USER?error} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD?error} + # pgBackRest + PGBACKREST_REPO1_TYPE: gcs + PGBACKREST_REPO1_GCS_KEY: /run/secrets/pgbackrest_gc_service_account_key + PGBACKREST_REPO1_GCS_BUCKET: ${PGBACKREST_REPO1_GCS_BUCKET?error} + PGBACKREST_REPO1_PATH: ${PGBACKREST_REPO1_PATH?error} + command: postgres -c archive_mode=on -c archive_command="pgbackrest --stanza=main archive-push %p" + volumes: + - ./data/postgres:/var/lib/postgresql/data + # pgBackrest + - ./data/pgbackrest/log/:/var/log/pgbackrest/ + - ./postgres/pgbackrest/pgbackrest.conf:/etc/pgbackrest/pgbackrest.conf:ro + post_start: + - command: /pgbackrest-setup.sh + user: root + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER"] + interval: 10s + timeout: 5s + retries: 5 + labels: + # NOTE: the cron format starts with seconds, instead of minutes. https://pkg.go.dev/github.com/robfig/cron?utm_source=godoc + ofelia.enabled: "true" + # TODO: Update this? + # NOTE: Make sure to align this with the retention policies ./postgres/pgbackrest/pgbackrest.conf + # Incremental Backups every day (At 22:00) + ofelia.job-exec.incr.schedule: "0 0 22 * * *" + ofelia.job-exec.incr.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main backup --type=incr" + ofelia.job-exec.incr.user: "postgres" + ofelia.job-exec.incr.no-overlap: 1 + # Differential backup every week (At 02:30 on Monday) + ofelia.job-exec.diff.schedule: "0 30 2 * * 1" + ofelia.job-exec.diff.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main backup --type=diff" + ofelia.job-exec.diff.user: "postgres" + ofelia.job-exec.diff.no-overlap: 1 + # Full backup every month (At 10:00 on day-of-month 1) + ofelia.job-exec.full.schedule: "0 0 10 1 * *" + ofelia.job-exec.full.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main backup --type=full" + ofelia.job-exec.full.user: "postgres" + ofelia.job-exec.full.no-overlap: 1 + # Expire every day (At 01:30) + ofelia.job-exec.expire.schedule: "0 30 01 * * *" + ofelia.job-exec.expire.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main expire" + ofelia.job-exec.expire.user: "postgres" + ofelia.job-exec.expire.no-overlap: 1 + # TODO: validate? + secrets: + - pgbackrest_gc_service_account_key + profiles: + - core + + # Redis alternative + dragonfly: + image: docker.dragonflydb.io/dragonflydb/dragonfly:v1.33.1 + restart: unless-stopped + ulimits: + memlock: -1 + volumes: + - dragonflydata:/data + profiles: + - core + + web: + <<: *base_server_setup + environment: + <<: *base_server_environments + APP_TYPE: "WEB" + command: bash -c "/code/misc/prod/run_web.sh" + + worker-beat: + <<: *base_worker_setup + command: bash -c "/code/misc/prod/run_worker_beat.sh" + + worker: + <<: *base_worker_setup + command: bash -c "/code/misc/prod/run_worker.sh" + + # Manager Dashboard + manager-dashboard: + build: + context: manager-dashboard/ + target: builder + tty: true + environment: + APP_ENVIRONMENT: ${MAPSWIPE_ENVIRONMENT:-PROD} + APP_COMMIT_HASH: ${MANAGER_DASHBOARD_COMMIT_HASH} + command: sh -c 'pnpm generate:type && pnpm build' + volumes: + - ./data/web-builds/manager-dashboard:/code/build + env_file: + - ./env/manager-dashboard.env + profiles: + - web-builds + + # Community Dashboard + community-dashboard: + build: + context: community-dashboard/ + target: builder + tty: true + environment: + APP_ENVIRONMENT: ${MAPSWIPE_ENVIRONMENT:-PROD} + APP_COMMIT_HASH: ${COMMUNITY_DASHBOARD_COMMIT_HASH} + command: sh -c 'pnpm generate:type && pnpm build' + volumes: + - ./data/web-builds/community-dashboard:/code/build + env_file: + - ./env/community-dashboard.env + profiles: + - web-builds + + # TODO: firebase deploy + + caddy: + image: caddy:2.10.2 + restart: unless-stopped + volumes: + # Caddy config + - ./caddy/Caddyfile:/etc/caddy/Caddyfile:ro + - ./data/caddy/data/:/data + - ./data/caddy/config/:/config + # Static files + # -- Backend + - ./data/backend/static:/assets/backend/static:ro + - ./data/backend/media:/assets/backend/media:ro + # -- Dashboards + - ./data/web-builds/manager-dashboard:/assets/manager-dashboard:ro + - ./data/web-builds/community-dashboard:/assets/community-dashboard:ro + environment: + CADDY_EMAIL: ${CADDY_EMAIL?error} + CADDY_HOST_BACKEND: ${CADDY_HOST_BACKEND?error} + CADDY_HOST_MANAGER_DASHBOARD: ${CADDY_HOST_MANAGER_DASHBOARD?error} + CADDY_HOST_COMMUNITY_DASHBOARD: ${CADDY_HOST_COMMUNITY_DASHBOARD?error} + ports: + - 80:80 + - 443:443 + depends_on: + - web + profiles: + - core + +volumes: + dragonflydata: + ipython_data_local: + +secrets: + pgbackrest_gc_service_account_key: + file: secrets/pgbackrest_gc_service_account_key.json diff --git a/env/.gitignore b/env/.gitignore new file mode 100644 index 0000000..7dc54a5 --- /dev/null +++ b/env/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything +* + +# But not this file +!.gitignore diff --git a/mapswipe-firebase b/firebase similarity index 100% rename from mapswipe-firebase rename to firebase diff --git a/manager-dashboard b/manager-dashboard index b68a146..4384bf1 160000 --- a/manager-dashboard +++ b/manager-dashboard @@ -1 +1 @@ -Subproject commit b68a1468f422e3ca3228e87e4769e0426d9db001 +Subproject commit 4384bf1b8d5328f8e8eb9ea35f53c56f768ff685 diff --git a/mapswipe-backend b/mapswipe-backend deleted file mode 160000 index 07c00fa..0000000 --- a/mapswipe-backend +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 07c00faf68d20e8c90b0bfd762ae92999107bb50 diff --git a/postgres/Dockerfile b/postgres/Dockerfile new file mode 100644 index 0000000..eee8732 --- /dev/null +++ b/postgres/Dockerfile @@ -0,0 +1,43 @@ +ARG POSTGIS_VERSION=17-3.5 # postgresqlVersion-postgisVersion + +FROM postgis/postgis:${POSTGIS_VERSION} AS pgbackrest-build + +ARG PGBACKREST_VERSION=2.55.1 + +WORKDIR /build + +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + wget \ + python3-distutils \ + meson \ + gcc \ + libpq-dev \ + libssl-dev \ + libxml2-dev \ + pkg-config \ + liblz4-dev \ + libzstd-dev \ + libbz2-dev \ + libz-dev \ + libyaml-dev \ + libssh2-1-dev + +RUN wget -q -O - \ + "https://github.com/pgbackrest/pgbackrest/archive/release/${PGBACKREST_VERSION}.tar.gz" | \ + tar zx -C ./ + +RUN meson setup pgbackrest "pgbackrest-release-${PGBACKREST_VERSION}" && \ + ninja -C pgbackrest + +# ---------------- Final image ------ +FROM postgis/postgis:$POSTGIS_VERSION AS postgis + +COPY pgbackrest/setup.sh /pgbackrest-setup.sh + +COPY --from=pgbackrest-build /build/pgbackrest/src/pgbackrest /usr/bin + +RUN chmod +x /usr/bin/pgbackrest && \ + pgbackrest version + +CMD ["postgres"] diff --git a/postgres/pgbackrest/pgbackrest.conf b/postgres/pgbackrest/pgbackrest.conf new file mode 100644 index 0000000..c9103a0 --- /dev/null +++ b/postgres/pgbackrest/pgbackrest.conf @@ -0,0 +1,22 @@ +# vim: filetype=toml + +[main] +pg1-path=/var/lib/postgresql/data/ + +[global] +# NOTE: Make sure to align this with the backup schedule docker-compose.yaml +# RETENTIONS TODO: UPDATE THIS? +# Keep 4 last full backups +repo1-retention-full=4 +# Keep 6 differential backups after the last full backup +repo1-retention-diff=6 +# Keep 7 days of WAL archives using incremental archive retention +repo1-retention-archive-type=incr +repo1-retention-archive=7 + +# Performance tuning +process-max=4 + +# Compression settings +compress-type=zst +compress-level=3 diff --git a/postgres/pgbackrest/setup.sh b/postgres/pgbackrest/setup.sh new file mode 100755 index 0000000..f918613 --- /dev/null +++ b/postgres/pgbackrest/setup.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +if [[ "$EUID" -ne 0 ]]; then + echo "Error: This script must be run as root." >&2 + exit 1 +fi + +set -xe + +# Configuration Directories +mkdir -p -m 770 /var/log/pgbackrest +chown postgres:postgres /var/log/pgbackrest diff --git a/scripts/get_commit_hash.sh b/scripts/get_commit_hash.sh new file mode 100755 index 0000000..476f936 --- /dev/null +++ b/scripts/get_commit_hash.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Set the base Git directory (default to current .git directory) +BASE_DIR="${1:-.git}" +HEAD_PATH="$BASE_DIR/HEAD" + +# Function to get the current commit SHA +if [[ ! -f "$HEAD_PATH" ]]; then + echo "HEAD file not found: $HEAD_PATH" >&2 + exit 1 +fi + +# Read the content of HEAD +ref=$(<"$HEAD_PATH") +ref=$(echo "$ref" | tr -d '\n') + +if [[ "$ref" == ref:\ * ]]; then + # It's a symbolic ref + ref_path="$BASE_DIR/${ref#ref: }" + if [[ -f "$ref_path" ]]; then + commit_hash=$(<"$ref_path") + echo "$commit_hash" + else + echo "Ref file not found: $ref_path" >&2 + exit 1 + fi +else + # Detached HEAD + echo "$ref" +fi diff --git a/secrets/.gitignore b/secrets/.gitignore new file mode 100644 index 0000000..7dc54a5 --- /dev/null +++ b/secrets/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything +* + +# But not this file +!.gitignore diff --git a/taskfile.yaml b/taskfile.yaml new file mode 100644 index 0000000..504eb17 --- /dev/null +++ b/taskfile.yaml @@ -0,0 +1,75 @@ +# TODO: Use this instead https://github.com/casey/just? + +version: '3' + +env: + MANAGER_DASHBOARD_COMMIT_HASH: + sh: ./scripts/get_commit_hash.sh .git/modules/manager-dashboard + COMMUNITY_DASHBOARD_COMMIT_HASH: + sh: ./scripts/get_commit_hash.sh .git/modules/community-dashboard + BACKEND_COMMIT_HASH: + sh: ./scripts/get_commit_hash.sh .git/modules/mapswipe-backend + +tasks: + # Web apps + web-build-manager: + cmds: + - docker compose --profile web-builds up --build --abort-on-container-exit manager-dashboard + + web-build-community: + cmds: + - docker compose --profile web-builds up --build --abort-on-container-exit community-dashboard + + web-builds: + cmds: + - task: web-build-community + - task: web-build-manager + - echo "Success" + + # Backend + backend-build-web: + cmds: + - docker compose --profile core build web + + backend-build-worker: + cmds: + - docker compose --profile core build worker + - docker compose --profile core build worker-beat + + backend-builds: + cmds: + - task: backend-build-web + - task: backend-build-worker + - echo "Success" + + backend-migration: + cmds: + - docker compose --profile core run --rm web ./manage.py migrate + + backend-collect-static: + cmds: + - docker compose --profile core run --rm web ./manage.py collectstatic --noinput + + backend-post-deploy: + cmds: + - task: backend-migration + - task: backend-collect-static + + backend-deploy: + cmds: + - task: backend-builds + - docker compose --profile core up -d web worker worker-beat + - task: backend-post-deploy + + # Misc + caddy-deploy: + cmd: docker compose --profile core up -d caddy + + deploy: + cmds: + - task: backend-deploy + - task: web-builds + - task: caddy-deploy + + logs: + cmd: docker compose --profile core logs -f --tail 1000 {{.CLI_ARGS}} diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 0000000..3fae00f --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1,3 @@ +.terragrunt-cache +.terraform +*.tfvars diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000..9d4b5e7 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,18 @@ +## Project ID + +```hcl +project_id_map = { + stage = "project-id-not-number" + prod = "project-id-not-number" +} +``` + +## Apply changes + +```bash +cd live/stage + +terragrunt plan + +terragrunt apply +``` diff --git a/terraform/live/.terraform.lock.hcl b/terraform/live/.terraform.lock.hcl new file mode 100644 index 0000000..da908d2 --- /dev/null +++ b/terraform/live/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.1.1" + constraints = "~> 7.0" + hashes = [ + "h1:+503QqsMOI/ckB+CAXYSm+WrYh2mqQNDKSNO6jISWeE=", + "zh:03ee9fdc0d157a606aba68658de6dc809fc3335cccb7c537373d8643412c1327", + "zh:110e8ffe81deb8c203ecf310a15c2dedca1dfc936473a247b8a4f98adebd86f5", + "zh:459e3419c004e7a475fb60cc52d47a34b3dc4e4de905eaa8e8f78ddbe550a9b5", + "zh:466cd31cee36877bc18aeabed80d1f4a22bac4e59a460be6e8bdb72dedca0e2b", + "zh:51d707eb2d854fa16dcbe21e29b01534eb893a2152a219ea84a15bbd87a4ff64", + "zh:69d6a1c83ffddd7f81273a98fb0ff7c13985a3c876565dd3df76c730c9929871", + "zh:9b5050da221735c7e8f75ed00d25578afaf8ed94a8c2f1f58f471eee98105d10", + "zh:ab01f2fd961ee86d99a55186093620d29f5323c0cd5613284d484e333679d70b", + "zh:d0f5b15774b15991baf71eb4a55a6831e3fb4b603f589f80b03393b46a9657a4", + "zh:dc198ec4b42435321f4fa12ca8d713cd350ff2f82d8749b87785b91b15b7c3ed", + "zh:e949c00ce89c92b7ed16cc0b0aed8e80d6416b240dc02047f9fa1de49aa4c44e", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/live/stage/.terraform.lock.hcl b/terraform/live/stage/.terraform.lock.hcl new file mode 100644 index 0000000..ae7074f --- /dev/null +++ b/terraform/live/stage/.terraform.lock.hcl @@ -0,0 +1,19 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/google" { + version = "7.1.1" + hashes = [ + "h1:XrEP4BdVzYGwdvMZU2xZqLe1Bgw9ziKUn51lOAJ0UGk=", + "zh:0417107a4ca6a33699e5c7d9b2c7f49f821524d0656c0953dcf733983731c423", + "zh:108112fcacfeabcb491510f4564331f44f5164fbe155532aafdb691535e34e44", + "zh:277548a6f9317211dcdd5e758c21b2317da93e0c3a7c412e0625b8a85ccd6583", + "zh:4b48d20d7d83a35c66adc7b5e564048208f004bbbec52311b076bce00cbd88d4", + "zh:55f58d6a62b96f3366a330c8740f52cf2b5e1e4d21636d19a08e8acacb6739de", + "zh:56e339a62b768a8342783b91630fbe15117b51ada0daaad648deeac813ee1137", + "zh:b1cd454d8c8eec6ee96dfa4e429d9f8981475f0d3257a64464887d9b639f027d", + "zh:b9c1c56007bf1f78a22fb0775a0f5b042aadb939329fce577fb73cc9685a1f28", + "zh:cac57e0e3bd1809c84fe843ebd19bef512230d8c55868fdc1fb452e94afde561", + "zh:edf882fdc4043cc7d635aab70ddfe56e2eec65d44bee40f4b0dd9688d8bdc265", + ] +} diff --git a/terraform/live/stage/terragrunt.hcl b/terraform/live/stage/terragrunt.hcl new file mode 100644 index 0000000..9132cb7 --- /dev/null +++ b/terraform/live/stage/terragrunt.hcl @@ -0,0 +1,10 @@ +include "root" { + path = find_in_parent_folders("terragrunt.root.hcl") +} + +terraform { + source = "../../resources" +} + +inputs = { +} diff --git a/terraform/live/terragrunt.root.hcl b/terraform/live/terragrunt.root.hcl new file mode 100644 index 0000000..9b2db85 --- /dev/null +++ b/terraform/live/terragrunt.root.hcl @@ -0,0 +1,25 @@ +locals { + env_name = path_relative_to_include() + secrets_config = jsondecode(read_tfvars_file("${get_terragrunt_dir()}/../secrets.auto.tfvars")) +} + +remote_state { + disable_init = tobool(get_env("DISABLE_INIT", "false")) # Used for CI lint + backend = "gcs" + generate = { + path = "backend.tf" + if_exists = "overwrite" + } + config = { + project = local.secrets_config.project_id_map[local.env_name] + bucket = "mapswipe-tf-${local.env_name}" + prefix = "terraform/${local.env_name}" + location = "EU" + } +} + +inputs = { + env_name = local.env_name + gcs_project_id = local.secrets_config.project_id_map[local.env_name] + gcs_region = "EU" +} diff --git a/terraform/resources/.terraform.lock.hcl b/terraform/resources/.terraform.lock.hcl new file mode 100644 index 0000000..1b9818a --- /dev/null +++ b/terraform/resources/.terraform.lock.hcl @@ -0,0 +1,20 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/google" { + version = "7.1.1" + constraints = "~> 7.0" + hashes = [ + "h1:XrEP4BdVzYGwdvMZU2xZqLe1Bgw9ziKUn51lOAJ0UGk=", + "zh:0417107a4ca6a33699e5c7d9b2c7f49f821524d0656c0953dcf733983731c423", + "zh:108112fcacfeabcb491510f4564331f44f5164fbe155532aafdb691535e34e44", + "zh:277548a6f9317211dcdd5e758c21b2317da93e0c3a7c412e0625b8a85ccd6583", + "zh:4b48d20d7d83a35c66adc7b5e564048208f004bbbec52311b076bce00cbd88d4", + "zh:55f58d6a62b96f3366a330c8740f52cf2b5e1e4d21636d19a08e8acacb6739de", + "zh:56e339a62b768a8342783b91630fbe15117b51ada0daaad648deeac813ee1137", + "zh:b1cd454d8c8eec6ee96dfa4e429d9f8981475f0d3257a64464887d9b639f027d", + "zh:b9c1c56007bf1f78a22fb0775a0f5b042aadb939329fce577fb73cc9685a1f28", + "zh:cac57e0e3bd1809c84fe843ebd19bef512230d8c55868fdc1fb452e94afde561", + "zh:edf882fdc4043cc7d635aab70ddfe56e2eec65d44bee40f4b0dd9688d8bdc265", + ] +} diff --git a/terraform/resources/postgres-backup.tf b/terraform/resources/postgres-backup.tf new file mode 100644 index 0000000..1d144d9 --- /dev/null +++ b/terraform/resources/postgres-backup.tf @@ -0,0 +1,32 @@ +resource "google_storage_bucket" "db_backup_bucket_name" { + name = "mapswipe-postgres-backups-${var.env_name}" + location = var.gcs_region + # storage_class = "NEARLINE" + + uniform_bucket_level_access = true + + lifecycle_rule { + action { + type = "Delete" + } + condition { + age = 7 + } + } + +} + +resource "google_service_account" "db_backup_sa" { + account_id = "db-backup-sa-${var.env_name}" + display_name = "DB Backup Service Account (pgBackRest) - ${var.env_name}" +} + +resource "google_storage_bucket_iam_member" "db_backup_writer" { + bucket = google_storage_bucket.db_backup_bucket_name.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${google_service_account.db_backup_sa.email}" +} + +output "service_account_email" { + value = google_service_account.db_backup_sa.email +} diff --git a/terraform/resources/providers.tf b/terraform/resources/providers.tf new file mode 100644 index 0000000..7ee7ec9 --- /dev/null +++ b/terraform/resources/providers.tf @@ -0,0 +1,4 @@ +provider "google" { + project = var.gcs_project_id + region = var.gcs_region +} diff --git a/terraform/resources/terraform.tf b/terraform/resources/terraform.tf new file mode 100644 index 0000000..7d7248b --- /dev/null +++ b/terraform/resources/terraform.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.10" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 7.0" + } + } +} diff --git a/terraform/resources/variables.tf b/terraform/resources/variables.tf new file mode 100644 index 0000000..effe679 --- /dev/null +++ b/terraform/resources/variables.tf @@ -0,0 +1,16 @@ +variable "env_name" { + description = "Mapswipe environment name" + type = string +} + +# GCS +variable "gcs_project_id" { + description = "GCS project id" + type = string + sensitive = true +} + +variable "gcs_region" { + description = "GCS region" + type = string +} diff --git a/terraform/terraform.tf b/terraform/terraform.tf new file mode 100644 index 0000000..4ca194f --- /dev/null +++ b/terraform/terraform.tf @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.9.0" + + backend "gcs" { + bucket = "tf-state" + prefix = "terraform/state" + } + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 7.0" + } + } +}