From c8f7d4cdb80e2a82bd7713025d25157176485f00 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Sun, 7 Sep 2025 11:43:10 +0545 Subject: [PATCH 01/10] chore: rename mapswipe-module to module --- .gitmodules | 4 ++-- mapswipe-backend => backend | 0 mapswipe-firebase => firebase | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename mapswipe-backend => backend (100%) rename mapswipe-firebase => firebase (100%) diff --git a/.gitmodules b/.gitmodules index 4898f16..69b120d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,8 +1,8 @@ [submodule "mapswipe-firebase"] - path = mapswipe-firebase + path = firebase url = git@github.com:mapswipe/mapswipe-firebase.git [submodule "mapswipe-backend"] - path = mapswipe-backend + path = backend url = git@github.com:mapswipe/mapswipe-backend.git [submodule "community-dashboard"] path = community-dashboard diff --git a/mapswipe-backend b/backend similarity index 100% rename from mapswipe-backend rename to backend diff --git a/mapswipe-firebase b/firebase similarity index 100% rename from mapswipe-firebase rename to firebase From a8a17d631a98bb156bec42755ab3b35f29c15b13 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Mon, 8 Sep 2025 21:36:18 +0545 Subject: [PATCH 02/10] feat(stage): add terraform (tofu) with terragrunt --- terraform/.gitignore | 3 +++ terraform/README.md | 18 +++++++++++++ terraform/live/.terraform.lock.hcl | 22 ++++++++++++++++ terraform/live/stage/.terraform.lock.hcl | 19 ++++++++++++++ terraform/live/stage/terragrunt.hcl | 10 ++++++++ terraform/live/terragrunt.root.hcl | 24 ++++++++++++++++++ terraform/resources/.terraform.lock.hcl | 20 +++++++++++++++ terraform/resources/postgres-backup.tf | 32 ++++++++++++++++++++++++ terraform/resources/providers.tf | 4 +++ terraform/resources/terraform.tf | 9 +++++++ terraform/resources/variables.tf | 16 ++++++++++++ terraform/terraform.tf | 15 +++++++++++ 12 files changed, 192 insertions(+) create mode 100644 terraform/.gitignore create mode 100644 terraform/README.md create mode 100644 terraform/live/.terraform.lock.hcl create mode 100644 terraform/live/stage/.terraform.lock.hcl create mode 100644 terraform/live/stage/terragrunt.hcl create mode 100644 terraform/live/terragrunt.root.hcl create mode 100644 terraform/resources/.terraform.lock.hcl create mode 100644 terraform/resources/postgres-backup.tf create mode 100644 terraform/resources/providers.tf create mode 100644 terraform/resources/terraform.tf create mode 100644 terraform/resources/variables.tf create mode 100644 terraform/terraform.tf diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 0000000..3fae00f --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1,3 @@ +.terragrunt-cache +.terraform +*.tfvars diff --git a/terraform/README.md b/terraform/README.md new file mode 100644 index 0000000..9d4b5e7 --- /dev/null +++ b/terraform/README.md @@ -0,0 +1,18 @@ +## Project ID + +```hcl +project_id_map = { + stage = "project-id-not-number" + prod = "project-id-not-number" +} +``` + +## Apply changes + +```bash +cd live/stage + +terragrunt plan + +terragrunt apply +``` diff --git a/terraform/live/.terraform.lock.hcl b/terraform/live/.terraform.lock.hcl new file mode 100644 index 0000000..da908d2 --- /dev/null +++ b/terraform/live/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "7.1.1" + constraints = "~> 7.0" + hashes = [ + "h1:+503QqsMOI/ckB+CAXYSm+WrYh2mqQNDKSNO6jISWeE=", + "zh:03ee9fdc0d157a606aba68658de6dc809fc3335cccb7c537373d8643412c1327", + "zh:110e8ffe81deb8c203ecf310a15c2dedca1dfc936473a247b8a4f98adebd86f5", + "zh:459e3419c004e7a475fb60cc52d47a34b3dc4e4de905eaa8e8f78ddbe550a9b5", + "zh:466cd31cee36877bc18aeabed80d1f4a22bac4e59a460be6e8bdb72dedca0e2b", + "zh:51d707eb2d854fa16dcbe21e29b01534eb893a2152a219ea84a15bbd87a4ff64", + "zh:69d6a1c83ffddd7f81273a98fb0ff7c13985a3c876565dd3df76c730c9929871", + "zh:9b5050da221735c7e8f75ed00d25578afaf8ed94a8c2f1f58f471eee98105d10", + "zh:ab01f2fd961ee86d99a55186093620d29f5323c0cd5613284d484e333679d70b", + "zh:d0f5b15774b15991baf71eb4a55a6831e3fb4b603f589f80b03393b46a9657a4", + "zh:dc198ec4b42435321f4fa12ca8d713cd350ff2f82d8749b87785b91b15b7c3ed", + "zh:e949c00ce89c92b7ed16cc0b0aed8e80d6416b240dc02047f9fa1de49aa4c44e", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/terraform/live/stage/.terraform.lock.hcl b/terraform/live/stage/.terraform.lock.hcl new file mode 100644 index 0000000..ae7074f --- /dev/null +++ b/terraform/live/stage/.terraform.lock.hcl @@ -0,0 +1,19 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/google" { + version = "7.1.1" + hashes = [ + "h1:XrEP4BdVzYGwdvMZU2xZqLe1Bgw9ziKUn51lOAJ0UGk=", + "zh:0417107a4ca6a33699e5c7d9b2c7f49f821524d0656c0953dcf733983731c423", + "zh:108112fcacfeabcb491510f4564331f44f5164fbe155532aafdb691535e34e44", + "zh:277548a6f9317211dcdd5e758c21b2317da93e0c3a7c412e0625b8a85ccd6583", + "zh:4b48d20d7d83a35c66adc7b5e564048208f004bbbec52311b076bce00cbd88d4", + "zh:55f58d6a62b96f3366a330c8740f52cf2b5e1e4d21636d19a08e8acacb6739de", + "zh:56e339a62b768a8342783b91630fbe15117b51ada0daaad648deeac813ee1137", + "zh:b1cd454d8c8eec6ee96dfa4e429d9f8981475f0d3257a64464887d9b639f027d", + "zh:b9c1c56007bf1f78a22fb0775a0f5b042aadb939329fce577fb73cc9685a1f28", + "zh:cac57e0e3bd1809c84fe843ebd19bef512230d8c55868fdc1fb452e94afde561", + "zh:edf882fdc4043cc7d635aab70ddfe56e2eec65d44bee40f4b0dd9688d8bdc265", + ] +} diff --git a/terraform/live/stage/terragrunt.hcl b/terraform/live/stage/terragrunt.hcl new file mode 100644 index 0000000..9132cb7 --- /dev/null +++ b/terraform/live/stage/terragrunt.hcl @@ -0,0 +1,10 @@ +include "root" { + path = find_in_parent_folders("terragrunt.root.hcl") +} + +terraform { + source = "../../resources" +} + +inputs = { +} diff --git a/terraform/live/terragrunt.root.hcl b/terraform/live/terragrunt.root.hcl new file mode 100644 index 0000000..69c6444 --- /dev/null +++ b/terraform/live/terragrunt.root.hcl @@ -0,0 +1,24 @@ +locals { + env_name = path_relative_to_include() + secrets_config = jsondecode(read_tfvars_file("${get_terragrunt_dir()}/../secrets.auto.tfvars")) +} + +remote_state { + backend = "gcs" + generate = { + path = "backend.tf" + if_exists = "overwrite" + } + config = { + project = local.secrets_config.project_id_map[local.env_name] + bucket = "mapswipe-tf-${local.env_name}" + prefix = "terraform/${local.env_name}" + location = "EU" + } +} + +inputs = { + env_name = local.env_name + gcs_project_id = local.secrets_config.project_id_map[local.env_name] + gcs_region = "EU" +} diff --git a/terraform/resources/.terraform.lock.hcl b/terraform/resources/.terraform.lock.hcl new file mode 100644 index 0000000..1b9818a --- /dev/null +++ b/terraform/resources/.terraform.lock.hcl @@ -0,0 +1,20 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/google" { + version = "7.1.1" + constraints = "~> 7.0" + hashes = [ + "h1:XrEP4BdVzYGwdvMZU2xZqLe1Bgw9ziKUn51lOAJ0UGk=", + "zh:0417107a4ca6a33699e5c7d9b2c7f49f821524d0656c0953dcf733983731c423", + "zh:108112fcacfeabcb491510f4564331f44f5164fbe155532aafdb691535e34e44", + "zh:277548a6f9317211dcdd5e758c21b2317da93e0c3a7c412e0625b8a85ccd6583", + "zh:4b48d20d7d83a35c66adc7b5e564048208f004bbbec52311b076bce00cbd88d4", + "zh:55f58d6a62b96f3366a330c8740f52cf2b5e1e4d21636d19a08e8acacb6739de", + "zh:56e339a62b768a8342783b91630fbe15117b51ada0daaad648deeac813ee1137", + "zh:b1cd454d8c8eec6ee96dfa4e429d9f8981475f0d3257a64464887d9b639f027d", + "zh:b9c1c56007bf1f78a22fb0775a0f5b042aadb939329fce577fb73cc9685a1f28", + "zh:cac57e0e3bd1809c84fe843ebd19bef512230d8c55868fdc1fb452e94afde561", + "zh:edf882fdc4043cc7d635aab70ddfe56e2eec65d44bee40f4b0dd9688d8bdc265", + ] +} diff --git a/terraform/resources/postgres-backup.tf b/terraform/resources/postgres-backup.tf new file mode 100644 index 0000000..1d144d9 --- /dev/null +++ b/terraform/resources/postgres-backup.tf @@ -0,0 +1,32 @@ +resource "google_storage_bucket" "db_backup_bucket_name" { + name = "mapswipe-postgres-backups-${var.env_name}" + location = var.gcs_region + # storage_class = "NEARLINE" + + uniform_bucket_level_access = true + + lifecycle_rule { + action { + type = "Delete" + } + condition { + age = 7 + } + } + +} + +resource "google_service_account" "db_backup_sa" { + account_id = "db-backup-sa-${var.env_name}" + display_name = "DB Backup Service Account (pgBackRest) - ${var.env_name}" +} + +resource "google_storage_bucket_iam_member" "db_backup_writer" { + bucket = google_storage_bucket.db_backup_bucket_name.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${google_service_account.db_backup_sa.email}" +} + +output "service_account_email" { + value = google_service_account.db_backup_sa.email +} diff --git a/terraform/resources/providers.tf b/terraform/resources/providers.tf new file mode 100644 index 0000000..7ee7ec9 --- /dev/null +++ b/terraform/resources/providers.tf @@ -0,0 +1,4 @@ +provider "google" { + project = var.gcs_project_id + region = var.gcs_region +} diff --git a/terraform/resources/terraform.tf b/terraform/resources/terraform.tf new file mode 100644 index 0000000..7d7248b --- /dev/null +++ b/terraform/resources/terraform.tf @@ -0,0 +1,9 @@ +terraform { + required_version = ">= 1.10" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 7.0" + } + } +} diff --git a/terraform/resources/variables.tf b/terraform/resources/variables.tf new file mode 100644 index 0000000..effe679 --- /dev/null +++ b/terraform/resources/variables.tf @@ -0,0 +1,16 @@ +variable "env_name" { + description = "Mapswipe environment name" + type = string +} + +# GCS +variable "gcs_project_id" { + description = "GCS project id" + type = string + sensitive = true +} + +variable "gcs_region" { + description = "GCS region" + type = string +} diff --git a/terraform/terraform.tf b/terraform/terraform.tf new file mode 100644 index 0000000..4ca194f --- /dev/null +++ b/terraform/terraform.tf @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.9.0" + + backend "gcs" { + bucket = "tf-state" + prefix = "terraform/state" + } + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 7.0" + } + } +} From 563e624ad84e5104c60a41a7070e59a5e662156b Mon Sep 17 00:00:00 2001 From: thenav56 Date: Mon, 8 Sep 2025 21:36:28 +0545 Subject: [PATCH 03/10] lint(terraform): add pre-commit config --- .pre-commit-config.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6d871b6 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,24 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-case-conflict + - id: detect-private-key + + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: "v1.100.0" + hooks: + - id: terraform_fmt + - id: terraform_tflint + - id: terragrunt_fmt + - id: terragrunt_validate_inputs + files: (/terragrunt\.hcl)$ + - id: terragrunt_validate + files: (/terragrunt\.hcl)$ + args: + - --hook-config=--tf-path=tofu + # - id: terraform_trivy + # - id: infracost_breakdown From 3f6b13af47d92464ee828f3ce02b3c83ade7e3db Mon Sep 17 00:00:00 2001 From: thenav56 Date: Mon, 8 Sep 2025 21:38:41 +0545 Subject: [PATCH 04/10] feat(postgres): setup postgres + pgbackrest docker --- postgres/Dockerfile | 43 +++++++++++++++++++++++++++++ postgres/pgbackrest/pgbackrest.conf | 22 +++++++++++++++ postgres/pgbackrest/setup.sh | 12 ++++++++ 3 files changed, 77 insertions(+) create mode 100644 postgres/Dockerfile create mode 100644 postgres/pgbackrest/pgbackrest.conf create mode 100755 postgres/pgbackrest/setup.sh diff --git a/postgres/Dockerfile b/postgres/Dockerfile new file mode 100644 index 0000000..eee8732 --- /dev/null +++ b/postgres/Dockerfile @@ -0,0 +1,43 @@ +ARG POSTGIS_VERSION=17-3.5 # postgresqlVersion-postgisVersion + +FROM postgis/postgis:${POSTGIS_VERSION} AS pgbackrest-build + +ARG PGBACKREST_VERSION=2.55.1 + +WORKDIR /build + +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends \ + wget \ + python3-distutils \ + meson \ + gcc \ + libpq-dev \ + libssl-dev \ + libxml2-dev \ + pkg-config \ + liblz4-dev \ + libzstd-dev \ + libbz2-dev \ + libz-dev \ + libyaml-dev \ + libssh2-1-dev + +RUN wget -q -O - \ + "https://github.com/pgbackrest/pgbackrest/archive/release/${PGBACKREST_VERSION}.tar.gz" | \ + tar zx -C ./ + +RUN meson setup pgbackrest "pgbackrest-release-${PGBACKREST_VERSION}" && \ + ninja -C pgbackrest + +# ---------------- Final image ------ +FROM postgis/postgis:$POSTGIS_VERSION AS postgis + +COPY pgbackrest/setup.sh /pgbackrest-setup.sh + +COPY --from=pgbackrest-build /build/pgbackrest/src/pgbackrest /usr/bin + +RUN chmod +x /usr/bin/pgbackrest && \ + pgbackrest version + +CMD ["postgres"] diff --git a/postgres/pgbackrest/pgbackrest.conf b/postgres/pgbackrest/pgbackrest.conf new file mode 100644 index 0000000..18d7c67 --- /dev/null +++ b/postgres/pgbackrest/pgbackrest.conf @@ -0,0 +1,22 @@ +# vim: filetype=toml + +[main] +pg1-path=/var/lib/postgresql/data/ + +[global] +# NOTE: Make sure to align this with the backup schedule docker-compose.yaml +# RETENTIONS TODO: UPDATE THIS? +# Keep 6 differential backups after the last full backup +repo1-retention-full=4 +# Keep 6 differential backups after the last full backup +repo1-retention-diff=6 +# Keep 7 days of WAL archives using incremental archive retention +repo1-retention-archive-type=incr +repo1-retention-archive=7 + +# Performance tuning +process-max=4 + +# Compression settings +compress-type=zst +compress-level=3 diff --git a/postgres/pgbackrest/setup.sh b/postgres/pgbackrest/setup.sh new file mode 100755 index 0000000..f918613 --- /dev/null +++ b/postgres/pgbackrest/setup.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +if [[ "$EUID" -ne 0 ]]; then + echo "Error: This script must be run as root." >&2 + exit 1 +fi + +set -xe + +# Configuration Directories +mkdir -p -m 770 /var/log/pgbackrest +chown postgres:postgres /var/log/pgbackrest From 923051442a56714adf13edd302dc36fce13699f5 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Mon, 8 Sep 2025 21:40:54 +0545 Subject: [PATCH 05/10] feat(postgres): docker-compose setup with ofelia --- .env.sample | 19 ++++++++++ .gitignore | 3 ++ README.md | 16 ++++++++ docker-compose.yaml | 91 +++++++++++++++++++++++++++++++++++++++++++++ env/.gitignore | 5 +++ secrets/.gitignore | 5 +++ 6 files changed, 139 insertions(+) create mode 100644 .env.sample create mode 100644 .gitignore create mode 100644 README.md create mode 100644 docker-compose.yaml create mode 100644 env/.gitignore create mode 100644 secrets/.gitignore diff --git a/.env.sample b/.env.sample new file mode 100644 index 0000000..0742a20 --- /dev/null +++ b/.env.sample @@ -0,0 +1,19 @@ +# vim: filetype=bash + +COMPOSE_PROFILES=core +COMPOSE_PROJECT_NAME=mapswipe-deploy + +# Ofelia +OFELIA_PROJECT_NAME=mapswipe-prod +OFELIA_SLACK_WEBHOOK= + +# Postgres +# NOTE: Password: Use openssl rand -base64 48 | tr -dc 'A-Za-z0-9_@#%^+=-' | head -c 24 +POSTGRES_USER= +POSTGRES_PASSWORD= + +# PgBackRest +# https://pgbackrest.org/command.html +# https://pgbackrest.org/configuration.html#introduction +PGBACKREST_REPO1_GCS_BUCKET=demo-bucket +PGBACKREST_REPO1_PATH=/demo-repo diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..30bda03 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.env* +!.env.sample +data/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..4075644 --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +## Setup + +### pgBackRest + +Create "main" stanza +```bash +docker compose exec -u postgres postgres pgbackrest --stanza=main stanza-create +``` + + +## pgBackRest + +View backup info +```bash +docker compose exec -u postgres postgres pgbackrest --stanza=main info +``` diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..72fe89b --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,91 @@ +name: ${COMPOSE_PROJECT_NAME?error} + +services: + ofelia: + image: mcuadros/ofelia:0.3.17 + command: daemon --docker -f label=com.docker.compose.project=${COMPOSE_PROJECT_NAME?error} + restart: unless-stopped + volumes: + - ./data/ofelia/:/logs + - /var/run/docker.sock:/var/run/docker.sock:ro + labels: + # File log + ofelia.save-folder: /logs + ofelia.save-only-on-error: 0 + # Slack + ofelia.slack-webhook: ${OFELIA_SLACK_WEBHOOK?error} + ofelia.slack-only-on-error: 1 + # Folder Cleanup + # Disk check (Every hour) + ofelia.job-local.cleanup.schedule: "@hourly" + ofelia.job-local.cleanup.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} find /logs/ -type f -mtime +7 -delete -print" + profiles: + - core + + # Backend + postgres: + build: + context: ./postgres/ + dockerfile: Dockerfile + target: postgis + restart: unless-stopped + ports: + - 127.0.0.1:${POSTGRES_DB_LOCAL_EXPOSE_PORT:-5432}:5432 + environment: + # NOTE: This config is only used for db first startup only + POSTGRES_DB: ${POSTGRES_DB:-mapswipe} + POSTGRES_USER: ${POSTGRES_USER?error} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD?error} + # pgBackRest + PGBACKREST_REPO1_TYPE: gcs + PGBACKREST_REPO1_GCS_KEY: /run/secrets/gc_service_account_key + PGBACKREST_REPO1_GCS_BUCKET: ${PGBACKREST_REPO1_GCS_BUCKET?error} + PGBACKREST_REPO1_PATH: ${PGBACKREST_REPO1_PATH?error} + command: postgres -c archive_mode=on -c archive_command="pgbackrest --stanza=main archive-push %p" + volumes: + - ./data/postgres:/var/lib/postgresql/data + # pgBackrest + - ./data/pgbackrest/log/:/var/log/pgbackrest/ + - ./postgres/pgbackrest/pgbackrest.conf:/etc/pgbackrest/pgbackrest.conf:ro + post_start: + - command: /pgbackrest-setup.sh + user: root + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER"] + interval: 10s + timeout: 5s + retries: 5 + labels: + # NOTE: the cron format starts with seconds, instead of minutes. https://pkg.go.dev/github.com/robfig/cron?utm_source=godoc + ofelia.enabled: "false" + # TODO: Update this? + # NOTE: Make sure to align this with the retention policies ./postgres/pgbackrest/pgbackrest.conf + # Incremental Backups every day (At 22:00) + ofelia.job-exec.incr.schedule: "0 0 22 * * *" + ofelia.job-exec.incr.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main backup --type=incr" + ofelia.job-exec.incr.user: "postgres" + ofelia.job-exec.incr.no-overlap: 1 + # Differential backup every week (At 02:30 on Monday) + ofelia.job-exec.diff.schedule: "0 30 2 * * 1" + ofelia.job-exec.diff.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main backup --type=diff" + ofelia.job-exec.diff.user: "postgres" + ofelia.job-exec.diff.no-overlap: 1 + # Full backup every month (At 10:00 on day-of-month 1) + ofelia.job-exec.full.schedule: "0 0 10 1 * *" + ofelia.job-exec.full.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main backup --type=full" + ofelia.job-exec.full.user: "postgres" + ofelia.job-exec.full.no-overlap: 1 + # Expire every day (At 01:30) + ofelia.job-exec.expire.schedule: "0 30 01 * * *" + ofelia.job-exec.expire.command: sh -c "_ENV=${OFELIA_PROJECT_NAME?error} pgbackrest --stanza=main expire" + ofelia.job-exec.expire.user: "postgres" + ofelia.job-exec.expire.no-overlap: 1 + # TODO: validate? + secrets: + - gc_service_account_key + profiles: + - core + +secrets: + gc_service_account_key: + file: secrets/gc_service_account_key.json diff --git a/env/.gitignore b/env/.gitignore new file mode 100644 index 0000000..7dc54a5 --- /dev/null +++ b/env/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything +* + +# But not this file +!.gitignore diff --git a/secrets/.gitignore b/secrets/.gitignore new file mode 100644 index 0000000..7dc54a5 --- /dev/null +++ b/secrets/.gitignore @@ -0,0 +1,5 @@ +# Ignore everything +* + +# But not this file +!.gitignore From 1eebf6bd589af26f854b08555267a367bb5e5f7a Mon Sep 17 00:00:00 2001 From: thenav56 Date: Mon, 8 Sep 2025 22:51:22 +0545 Subject: [PATCH 06/10] feat(web-app): setup manager and community dashboard with Caddy - add script to get submodule commit hash - setup taskfile for web-app deployments --- .env.sample | 5 ++++ caddy/Caddyfile | 23 +++++++++++++++ community-dashboard | 2 +- docker-compose.yaml | 57 ++++++++++++++++++++++++++++++++++++++ manager-dashboard | 2 +- scripts/get_commit_hash.sh | 30 ++++++++++++++++++++ taskfile.yaml | 26 +++++++++++++++++ 7 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 caddy/Caddyfile create mode 100755 scripts/get_commit_hash.sh create mode 100644 taskfile.yaml diff --git a/.env.sample b/.env.sample index 0742a20..eff1e08 100644 --- a/.env.sample +++ b/.env.sample @@ -12,6 +12,11 @@ OFELIA_SLACK_WEBHOOK= POSTGRES_USER= POSTGRES_PASSWORD= +# Caddy +CADDY_EMAIL=mapswipe@example.org +CADDY_HOST_MANAGER_DASHBOARD=https://manager.example.org +CADDY_HOST_COMMUNITY_DASHBOARD=https://community.example.org + # PgBackRest # https://pgbackrest.org/command.html # https://pgbackrest.org/configuration.html#introduction diff --git a/caddy/Caddyfile b/caddy/Caddyfile new file mode 100644 index 0000000..49c5208 --- /dev/null +++ b/caddy/Caddyfile @@ -0,0 +1,23 @@ +(file_server_config) { + encode gzip + file_server +} + + +{$CADDY_HOST_MANAGER_DASHBOARD} { + handle { + try_files {path} /index.html + root * /assets/manager-dashboard + import file_server_config + } +} + + +{$CADDY_HOST_COMMUNITY_DASHBOARD} { + handle { + try_files {path} /index.html + root * /assets/manager-dashboard + import file_server_config + } +} + diff --git a/community-dashboard b/community-dashboard index 9892645..0680edf 160000 --- a/community-dashboard +++ b/community-dashboard @@ -1 +1 @@ -Subproject commit 9892645ee15d24f155bf3810151e4d5491b4a375 +Subproject commit 0680edf49d2870866097fb58abbd9c0246d6cbdc diff --git a/docker-compose.yaml b/docker-compose.yaml index 72fe89b..eb13f3d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -86,6 +86,63 @@ services: profiles: - core + + # Manager Dashboard + manager-dashboard: + build: + context: manager-dashboard/ + target: builder + tty: true + environment: + APP_ENVIRONMENT: ${MAPSWIPE_ENVIRONMENT:-PROD} + APP_COMMIT_HASH: ${MANAGER_DASHBOARD_COMMIT_HASH} + command: sh -c 'pnpm generate:type && pnpm build' + volumes: + - ./data/web-builds/manager-dashboard:/code/build + env_file: + - ./env/manager-dashboard.env + profiles: + - web-builds + + # Community Dashboard + community-dashboard: + build: + context: community-dashboard/ + target: builder + tty: true + environment: + APP_ENVIRONMENT: ${MAPSWIPE_ENVIRONMENT:-PROD} + APP_COMMIT_HASH: ${COMMUNITY_DASHBOARD_COMMIT_HASH} + command: sh -c 'pnpm generate:type && pnpm build' + volumes: + - ./data/web-builds/community-dashboard:/code/build + env_file: + - ./env/community-dashboard.env + profiles: + - web-builds + + caddy: + image: caddy:2.10.2 + restart: unless-stopped + volumes: + # Caddy config + - ./caddy/Caddyfile:/etc/caddy/Caddyfile:ro + - ./data/caddy/data/:/data + - ./data/caddy/config/:/config + # Static files + # -- Dashboards + - ./data/web-builds/manager-dashboard:/assets/manager-dashboard:ro + - ./data/web-builds/community-dashboard:/assets/community-dashboard:ro + environment: + CADDY_EMAIL: ${CADDY_EMAIL?error} + CADDY_HOST_MANAGER_DASHBOARD: ${CADDY_HOST_MANAGER_DASHBOARD?error} + CADDY_HOST_COMMUNITY_DASHBOARD: ${CADDY_HOST_COMMUNITY_DASHBOARD?error} + ports: + - 80:80 + - 443:443 + profiles: + - core + secrets: gc_service_account_key: file: secrets/gc_service_account_key.json diff --git a/manager-dashboard b/manager-dashboard index b68a146..0a04216 160000 --- a/manager-dashboard +++ b/manager-dashboard @@ -1 +1 @@ -Subproject commit b68a1468f422e3ca3228e87e4769e0426d9db001 +Subproject commit 0a04216cd005ae56b56539968ce9f8e794699964 diff --git a/scripts/get_commit_hash.sh b/scripts/get_commit_hash.sh new file mode 100755 index 0000000..476f936 --- /dev/null +++ b/scripts/get_commit_hash.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Set the base Git directory (default to current .git directory) +BASE_DIR="${1:-.git}" +HEAD_PATH="$BASE_DIR/HEAD" + +# Function to get the current commit SHA +if [[ ! -f "$HEAD_PATH" ]]; then + echo "HEAD file not found: $HEAD_PATH" >&2 + exit 1 +fi + +# Read the content of HEAD +ref=$(<"$HEAD_PATH") +ref=$(echo "$ref" | tr -d '\n') + +if [[ "$ref" == ref:\ * ]]; then + # It's a symbolic ref + ref_path="$BASE_DIR/${ref#ref: }" + if [[ -f "$ref_path" ]]; then + commit_hash=$(<"$ref_path") + echo "$commit_hash" + else + echo "Ref file not found: $ref_path" >&2 + exit 1 + fi +else + # Detached HEAD + echo "$ref" +fi diff --git a/taskfile.yaml b/taskfile.yaml new file mode 100644 index 0000000..2e8192f --- /dev/null +++ b/taskfile.yaml @@ -0,0 +1,26 @@ +# TODO: Use this instead https://github.com/casey/just? +# TODO: Add this to README? go install github.com/go-task/task/v3/cmd/task@latest + +version: '3' + +env: + MANAGER_DASHBOARD_COMMIT_HASH: + sh: ./scripts/get_commit_hash.sh .git/modules/manager-dashboard + COMMUNITY_DASHBOARD_COMMIT_HASH: + sh: ./scripts/get_commit_hash.sh .git/modules/community-dashboard + +tasks: + # Web apps + web-build-manager: + cmds: + - docker compose --profile web-builds up --build --abort-on-container-exit manager-dashboard + + web-build-community: + cmds: + - docker compose --profile web-builds up --build --abort-on-container-exit community-dashboard + + web-builds: + cmds: + - task: web-build-community + - task: web-build-manager + - echo "Success" From f0dd378b7b1fa8e0ecdeaefc97443d9739b73f8c Mon Sep 17 00:00:00 2001 From: thenav56 Date: Mon, 8 Sep 2025 23:08:58 +0545 Subject: [PATCH 07/10] lint: additional pre-commit hooks --- .pre-commit-config.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d871b6..7eae012 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,15 @@ repos: + - repo: https://github.com/gitleaks/gitleaks + rev: v8.24.2 + hooks: + - id: gitleaks + + - repo: https://github.com/crate-ci/typos + rev: v1.31.1 + hooks: + - id: typos + args: ["--exclude=CHANGELOG.md", "--force-exclude"] + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: @@ -7,6 +18,16 @@ repos: - id: check-yaml - id: check-case-conflict - id: detect-private-key + - id: check-merge-conflict + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-json + - id: check-toml + - id: check-xml + - id: check-yaml + args: [--unsafe] + - id: debug-statements + - id: detect-private-key - repo: https://github.com/antonbabenko/pre-commit-terraform rev: "v1.100.0" From 8f7d73ed10d73d1f26bc28bafb325e655814fd83 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Mon, 8 Sep 2025 23:12:20 +0545 Subject: [PATCH 08/10] lint: add pre-commit github workflow --- .github/workflows/pre-commit.yaml | 47 ++++++++++++++++++++++++++++++ caddy/Caddyfile | 1 - terraform/live/terragrunt.root.hcl | 3 +- 3 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/pre-commit.yaml diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 0000000..ef5e746 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,47 @@ +name: Pre commit + +on: + push: + +jobs: + pre_commit_checks: + name: Pre-Commit checks + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@main + with: + submodules: true + + - name: Generate fake ./terraform/live/secrets.auto.tfvars + run: | + echo "project_id_map = { stage = \"mapswipe\" }" > ./terraform/live/secrets.auto.tfvars + + # TODO: Cache plugins? + - uses: terraform-linters/setup-tflint@v5 + name: Setup TFLint + with: + tflint_version: v0.52.0 + + - name: Install Terragrunt and OpenTofu + uses: gruntwork-io/terragrunt-action@v3 + with: + # TODO: Use mise instead? https://github.com/gruntwork-io/terragrunt-action#tool-version-management + tg_version: 0.80.4 + tofu_version: v1.10.5 + + - name: Initialize tofu without any backend + run: | + # Find all directories containing terragrunt.hcl + find . -type f -name "terragrunt.hcl" | while read hcl_file; do + dir=$(dirname "$hcl_file") + echo "Entering directory: $dir" + ( + cd "$dir" || exit + tofu init -backend=false + ) + done + + - uses: pre-commit/action@main + env: + DISABLE_INIT: true diff --git a/caddy/Caddyfile b/caddy/Caddyfile index 49c5208..b6c0852 100644 --- a/caddy/Caddyfile +++ b/caddy/Caddyfile @@ -20,4 +20,3 @@ import file_server_config } } - diff --git a/terraform/live/terragrunt.root.hcl b/terraform/live/terragrunt.root.hcl index 69c6444..9b2db85 100644 --- a/terraform/live/terragrunt.root.hcl +++ b/terraform/live/terragrunt.root.hcl @@ -4,7 +4,8 @@ locals { } remote_state { - backend = "gcs" + disable_init = tobool(get_env("DISABLE_INIT", "false")) # Used for CI lint + backend = "gcs" generate = { path = "backend.tf" if_exists = "overwrite" From c8e64c896b3b2a0c8a08f95b6cdd2ee7659a18d4 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Tue, 9 Sep 2025 11:11:16 +0545 Subject: [PATCH 09/10] feat(backend): setup web/worker and caddy - update taskfile with more alias --- .env.sample | 1 + backend | 2 +- caddy/Caddyfile | 23 ++++++++++- docker-compose.yaml | 95 +++++++++++++++++++++++++++++++++++++++++++++ taskfile.yaml | 49 +++++++++++++++++++++++ 5 files changed, 168 insertions(+), 2 deletions(-) diff --git a/.env.sample b/.env.sample index eff1e08..d4d0091 100644 --- a/.env.sample +++ b/.env.sample @@ -14,6 +14,7 @@ POSTGRES_PASSWORD= # Caddy CADDY_EMAIL=mapswipe@example.org +CADDY_HOST_BACKEND=https://backend.example.org CADDY_HOST_MANAGER_DASHBOARD=https://manager.example.org CADDY_HOST_COMMUNITY_DASHBOARD=https://community.example.org diff --git a/backend b/backend index 07c00fa..40ff253 160000 --- a/backend +++ b/backend @@ -1 +1 @@ -Subproject commit 07c00faf68d20e8c90b0bfd762ae92999107bb50 +Subproject commit 40ff253ebacab2fbccb614d15fddce530732c829 diff --git a/caddy/Caddyfile b/caddy/Caddyfile index b6c0852..8529c28 100644 --- a/caddy/Caddyfile +++ b/caddy/Caddyfile @@ -4,6 +4,27 @@ } +{$CADDY_HOST_BACKEND} { + request_body { + max_size 10MB + } + + handle_path /static/* { + root * /assets/backend/static + import file_server_config + } + + handle_path /media/* { + root * /assets/backend/media + import file_server_config + } + + handle { + reverse_proxy http://web:80 + } +} + + {$CADDY_HOST_MANAGER_DASHBOARD} { handle { try_files {path} /index.html @@ -16,7 +37,7 @@ {$CADDY_HOST_COMMUNITY_DASHBOARD} { handle { try_files {path} /index.html - root * /assets/manager-dashboard + root * /assets/community-dashboard import file_server_config } } diff --git a/docker-compose.yaml b/docker-compose.yaml index eb13f3d..a099d07 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,65 @@ name: ${COMPOSE_PROJECT_NAME?error} +x-server: &base_server_setup + build: + context: ./backend/ + tags: + - mapswipe/mapswipe-backend:${BACKEND_DOCKER_TAG:-latest} + # Used for python debugging. + stdin_open: true + tty: true + restart: unless-stopped + environment: &base_server_environments + APP_RELEASE: ${BACKEND_COMMIT_HASH} + DEBUG: ${BACKEND_DEBUG:-false} + ENABLE_DEBUG_TOOLBAR: ${BACKEND_ENABLE_DEBUG_TOOLBAR:-false} + APP_ENVIRONMENT: ${MAPSWIPE_ENVIRONMENT:-PROD} + ENABLE_STRAWBERRY_GRAPHIQL: ${BACKEND_ENABLE_STRAWBERRY_GRAPHIQL:-false} + # Postgres + POSTGRES_HOST: ${POSTGRES_HOST:-postgres} + POSTGRES_PORT: ${POSTGRES_PORT:-5432} + POSTGRES_DB: ${POSTGRES_DB:-mapswipe} + POSTGRES_USER: ${POSTGRES_USER?error} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD?error} + # Redis (Dragonfly) + CELERY_REDIS_URL: ${BACKEND_CELERY_REDIS_URL:-redis://dragonfly:6379/0} + CACHE_REDIS_URL: ${BACKEND_CACHE_REDIS_URL:-redis://dragonfly:6379/1} + TEST_CACHE_REDIS_URL: ${BACKEND_TEST_CACHE_REDIS_URL:-redis://dragonfly:6379/11} + # Email (TODO: Dummy config) + EMAIL_HOST: ${BACKEND_EMAIL_HOST:-dummy} + EMAIL_PORT: ${BACKEND_EMAIL_PORT:-1025} + EMAIL_USE_TLS: ${BACKEND_EMAIL_USE_TLS:-false} + EMAIL_HOST_USER: ${BACKEND_EMAIL_HOST_USER:-dummy} + EMAIL_HOST_PASSWORD: ${BACKEND_EMAIL_HOST_PASSWORD:-dummy} + DEFAULT_FROM_EMAIL: ${BACKEND_DEFAULT_FROM_EMAIL:-Mapswipe Dev } + # Firebase + FIREBASE_EMULATOR_USE: false + # Storage + MEDIA_ROOT: "/data/media" + STATIC_ROOT: "/data/static" + MEDIA_URL: "media/" + STATIC_URL: "static/" + env_file: + - ./env/backend.env + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - ./data/backend/static:/data/static + - ./data/backend/media:/data/media + - ipython_data_local:/root/.ipython/profile_default # persist ipython data, including ipython history + depends_on: + - postgres + - dragonfly + profiles: + - core + + +x-worker: &base_worker_setup + <<: *base_server_setup + environment: + <<: *base_server_environments + APP_TYPE: "WORKER-BEAT" + services: ofelia: image: mcuadros/ofelia:0.3.17 @@ -86,6 +146,31 @@ services: profiles: - core + # Redis alternative + dragonfly: + image: docker.dragonflydb.io/dragonflydb/dragonfly:v1.33.1 + restart: unless-stopped + ulimits: + memlock: -1 + volumes: + - dragonflydata:/data + profiles: + - core + + web: + <<: *base_server_setup + environment: + <<: *base_server_environments + APP_TYPE: "WEB" + command: bash -c "/code/misc/prod/run_web.sh" + + worker-beat: + <<: *base_worker_setup + command: bash -c "/code/misc/prod/run_worker_beat.sh" + + worker: + <<: *base_worker_setup + command: bash -c "/code/misc/prod/run_worker.sh" # Manager Dashboard manager-dashboard: @@ -130,19 +215,29 @@ services: - ./data/caddy/data/:/data - ./data/caddy/config/:/config # Static files + # -- Backend + - ./data/backend/static:/assets/backend/static:ro + - ./data/backend/media:/assets/backend/media:ro # -- Dashboards - ./data/web-builds/manager-dashboard:/assets/manager-dashboard:ro - ./data/web-builds/community-dashboard:/assets/community-dashboard:ro environment: CADDY_EMAIL: ${CADDY_EMAIL?error} + CADDY_HOST_BACKEND: ${CADDY_HOST_BACKEND?error} CADDY_HOST_MANAGER_DASHBOARD: ${CADDY_HOST_MANAGER_DASHBOARD?error} CADDY_HOST_COMMUNITY_DASHBOARD: ${CADDY_HOST_COMMUNITY_DASHBOARD?error} ports: - 80:80 - 443:443 + depends_on: + - web profiles: - core +volumes: + dragonflydata: + ipython_data_local: + secrets: gc_service_account_key: file: secrets/gc_service_account_key.json diff --git a/taskfile.yaml b/taskfile.yaml index 2e8192f..1233b56 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -8,6 +8,8 @@ env: sh: ./scripts/get_commit_hash.sh .git/modules/manager-dashboard COMMUNITY_DASHBOARD_COMMIT_HASH: sh: ./scripts/get_commit_hash.sh .git/modules/community-dashboard + BACKEND_COMMIT_HASH: + sh: ./scripts/get_commit_hash.sh .git/modules/mapswipe-backend tasks: # Web apps @@ -24,3 +26,50 @@ tasks: - task: web-build-community - task: web-build-manager - echo "Success" + + # Backend + backend-build-web: + cmds: + - docker compose --profile core build web + + backend-build-worker: + cmds: + - docker compose --profile core build worker + - docker compose --profile core build worker-beat + + backend-builds: + cmds: + - task: backend-build-web + - task: backend-build-worker + - echo "Success" + + backend-deploy: + cmds: + - task: backend-builds + - docker compose --profile core up -d web worker worker-beat + + backend-migration: + cmds: + - docker compose --profile core run --rm web ./manage.py migrate + + backend-collect-static: + cmds: + - docker compose --profile core run --rm web ./manage.py collectstatic --noinput + + backend-post-deploy: + cmds: + - task: backend-migration + - task: backend-collect-static + + # Misc + caddy-deploy: + cmd: docker compose --profile core up -d caddy + + logs: + cmd: docker compose --profile core logs -f --tail 1000 {{.CLI_ARGS}} + + deploy: + cmds: + - task: backend-deploy + - task: web-builds + - task: caddy-deploy From 527d07e9660f4786494d10d33d6d493d6fd85d45 Mon Sep 17 00:00:00 2001 From: thenav56 Date: Tue, 9 Sep 2025 20:12:33 +0545 Subject: [PATCH 10/10] docs: deployment --- .env.sample | 1 + README.md | 58 +++++++++++++++++++++++++++-- backend | 2 +- community-dashboard | 2 +- docker-compose.yaml | 12 +++--- manager-dashboard | 2 +- postgres/pgbackrest/pgbackrest.conf | 2 +- taskfile.yaml | 18 ++++----- 8 files changed, 76 insertions(+), 21 deletions(-) diff --git a/.env.sample b/.env.sample index d4d0091..8d26bfc 100644 --- a/.env.sample +++ b/.env.sample @@ -2,6 +2,7 @@ COMPOSE_PROFILES=core COMPOSE_PROJECT_NAME=mapswipe-deploy +MAPSWIPE_ENVIRONMENT=prod|stage # Ofelia OFELIA_PROJECT_NAME=mapswipe-prod diff --git a/README.md b/README.md index 4075644..5f97492 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,60 @@ ## Setup +### Cloning the Repository + +Most submodules use SSH URLs. To avoid setting up SSH keys, run this command to use HTTPS instead: + +```bash +git config --global url."https://github.com/".insteadOf "git@github.com:" +``` + +Clone and pull all submodules +```bash +git clone git@github.com:mapswipe/mapswipe-deploy.git +cd mapswipe-deploy +git submodule update --init --recursive +``` + +### Environment Variables + +Make sure these environment files are in place: + +- .env (based on `.env.sample`) +- env/backend.env +- env/community-dashboard.env +- env/manager-dashboard.env +- secrets/pgbackrest_gc_service_account_key.json + +```bash +cp .env.sample .env +touch env/backend.env +touch env/community-dashboard.env +touch env/manager-dashboard.env +touch secrets/pgbackrest_gc_service_account_key.json +``` + +## Apply changes + +The `task` tool is used to set up a pre-alias. +> https://taskfile.dev/ + + +```bash +task --list-all + +# Deploy all +task deploy + +# Deploy web apps +task web-builds + +# Deploy backend resources +task backend-deploy + +# Deploy caddy +task caddy-deploy +``` + ### pgBackRest Create "main" stanza @@ -7,9 +62,6 @@ Create "main" stanza docker compose exec -u postgres postgres pgbackrest --stanza=main stanza-create ``` - -## pgBackRest - View backup info ```bash docker compose exec -u postgres postgres pgbackrest --stanza=main info diff --git a/backend b/backend index 40ff253..126ed64 160000 --- a/backend +++ b/backend @@ -1 +1 @@ -Subproject commit 40ff253ebacab2fbccb614d15fddce530732c829 +Subproject commit 126ed6431298d79c90cd46b2fc25b745f1ebb12a diff --git a/community-dashboard b/community-dashboard index 0680edf..ced6945 160000 --- a/community-dashboard +++ b/community-dashboard @@ -1 +1 @@ -Subproject commit 0680edf49d2870866097fb58abbd9c0246d6cbdc +Subproject commit ced6945483d3dedeb580dcdde21ad34211daa24c diff --git a/docker-compose.yaml b/docker-compose.yaml index a099d07..05adae4 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -98,7 +98,7 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD?error} # pgBackRest PGBACKREST_REPO1_TYPE: gcs - PGBACKREST_REPO1_GCS_KEY: /run/secrets/gc_service_account_key + PGBACKREST_REPO1_GCS_KEY: /run/secrets/pgbackrest_gc_service_account_key PGBACKREST_REPO1_GCS_BUCKET: ${PGBACKREST_REPO1_GCS_BUCKET?error} PGBACKREST_REPO1_PATH: ${PGBACKREST_REPO1_PATH?error} command: postgres -c archive_mode=on -c archive_command="pgbackrest --stanza=main archive-push %p" @@ -117,7 +117,7 @@ services: retries: 5 labels: # NOTE: the cron format starts with seconds, instead of minutes. https://pkg.go.dev/github.com/robfig/cron?utm_source=godoc - ofelia.enabled: "false" + ofelia.enabled: "true" # TODO: Update this? # NOTE: Make sure to align this with the retention policies ./postgres/pgbackrest/pgbackrest.conf # Incremental Backups every day (At 22:00) @@ -142,7 +142,7 @@ services: ofelia.job-exec.expire.no-overlap: 1 # TODO: validate? secrets: - - gc_service_account_key + - pgbackrest_gc_service_account_key profiles: - core @@ -206,6 +206,8 @@ services: profiles: - web-builds + # TODO: firebase deploy + caddy: image: caddy:2.10.2 restart: unless-stopped @@ -239,5 +241,5 @@ volumes: ipython_data_local: secrets: - gc_service_account_key: - file: secrets/gc_service_account_key.json + pgbackrest_gc_service_account_key: + file: secrets/pgbackrest_gc_service_account_key.json diff --git a/manager-dashboard b/manager-dashboard index 0a04216..4384bf1 160000 --- a/manager-dashboard +++ b/manager-dashboard @@ -1 +1 @@ -Subproject commit 0a04216cd005ae56b56539968ce9f8e794699964 +Subproject commit 4384bf1b8d5328f8e8eb9ea35f53c56f768ff685 diff --git a/postgres/pgbackrest/pgbackrest.conf b/postgres/pgbackrest/pgbackrest.conf index 18d7c67..c9103a0 100644 --- a/postgres/pgbackrest/pgbackrest.conf +++ b/postgres/pgbackrest/pgbackrest.conf @@ -6,7 +6,7 @@ pg1-path=/var/lib/postgresql/data/ [global] # NOTE: Make sure to align this with the backup schedule docker-compose.yaml # RETENTIONS TODO: UPDATE THIS? -# Keep 6 differential backups after the last full backup +# Keep 4 last full backups repo1-retention-full=4 # Keep 6 differential backups after the last full backup repo1-retention-diff=6 diff --git a/taskfile.yaml b/taskfile.yaml index 1233b56..504eb17 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -1,5 +1,4 @@ # TODO: Use this instead https://github.com/casey/just? -# TODO: Add this to README? go install github.com/go-task/task/v3/cmd/task@latest version: '3' @@ -43,11 +42,6 @@ tasks: - task: backend-build-worker - echo "Success" - backend-deploy: - cmds: - - task: backend-builds - - docker compose --profile core up -d web worker worker-beat - backend-migration: cmds: - docker compose --profile core run --rm web ./manage.py migrate @@ -61,15 +55,21 @@ tasks: - task: backend-migration - task: backend-collect-static + backend-deploy: + cmds: + - task: backend-builds + - docker compose --profile core up -d web worker worker-beat + - task: backend-post-deploy + # Misc caddy-deploy: cmd: docker compose --profile core up -d caddy - logs: - cmd: docker compose --profile core logs -f --tail 1000 {{.CLI_ARGS}} - deploy: cmds: - task: backend-deploy - task: web-builds - task: caddy-deploy + + logs: + cmd: docker compose --profile core logs -f --tail 1000 {{.CLI_ARGS}}