diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..5a6d7521d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,8 @@ +{ + "name": "OpenCodelists", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": {} + }, + "image": "mcr.microsoft.com/devcontainers/universal:3", + "postCreateCommand": "containerWorkspaceFolder=${containerWorkspaceFolder} ./local-deploy.sh" +} diff --git a/.dockerignore b/.dockerignore index 75e2e69f3..c730816e7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,3 +15,7 @@ node_modules/ staticfiles/ # dockerfile image build its own dir assets/dist/ + +# Database files +**/*.sqlite* +db.sqlite* diff --git a/LOCAL_TEST_DEPLOYMENT.md b/LOCAL_TEST_DEPLOYMENT.md new file mode 100644 index 000000000..f6819c130 --- /dev/null +++ b/LOCAL_TEST_DEPLOYMENT.md @@ -0,0 +1,75 @@ +# Testing Deployment Changes in the Devcontainer + +This guide explains how to test changes to deployment configuration (like the Procfile, environment variables, or other Dokku settings) locally before deploying to production. + +## Initial Setup + +Start the devcontainer either in VS Code or GitHub Codespaces. The devcontainer is configured to run a local Dokku instance, allowing you to test deployment changes without affecting the production environment. + +When the devcontainer starts, the `local-deploy.sh` script automatically: + +1. Sets up necessary dependencies (`just`) +2. Creates required directories and permissions +3. Starts Dokku and otel-collector containers +4. Builds the app image and deploys it to Dokku + +This provides a local environment that replicates the production Dokku setup. + +## Testing Deployment Changes + +### Workflow + +1. **Make your changes** + - Edit the `Procfile`, Dockerfile, or other deployment configuration + +2. **Run CI checks (optional but recommended)** + - Run `./local-ci-pipeline.sh` to execute all the CI checks locally + - This runs the same checks as GitHub Actions workflows (linting, tests, migrations) + - Note: The CI pipeline will run smoke tests on port 7001 to avoid conflicts with Dokku + +3. **Redeploy to local Dokku** + - Run `./local-redeploy.sh` to build, test, and deploy your changes + - Run as `RUN_CI=1 ./local-redeploy.sh` to run the full CI pipeline before deploying + +4. **Verify the changes** + - Access the app at http://localhost:7000 + - Check logs with `docker exec -t dokku sh -c "dokku logs opencodelists -t"` + +### Testing Remote Branches + +You can also test deployment changes from any remote branch: + +```bash +# Test deployment of a specific remote branch +./local-redeploy.sh branch-name + +# Test deployment of a specific remote branch with full CI pipeline +RUN_CI=1 ./local-redeploy.sh branch-name +``` + +The script will: +1. Fetch the remote branch if it doesn't exist locally +2. Create a temporary workspace with the branch's code +3. Run the deployment process on that branch's code +4. Clean up the temporary workspace automatically when done + +This allows you to test deployment changes from pull requests or experimental branches. + +### Port Configurations + +The testing environment uses the following port configurations: +- Dokku deployment: Runs on port 7000 +- CI smoke tests: Run on port 7001 to avoid conflicts with the Dokku deployment + +## Troubleshooting + +### Database Locks +Occasionally the smoke test fails because the database is locked. This typically happens when multiple processes try to access the SQLite database simultaneously. If this occurs: +- Wait a few seconds and run the script again +- The test environment uses separate storage directories to prevent conflicts + +### Branch Checkout Issues +If you encounter issues checking out a specific branch: +- Ensure the branch exists on the remote (run `git fetch --all` first) +- Check that you have permissions to access the branch +- The script will automatically create a local copy of remote branches diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 000000000..1174574c8 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,24 @@ +services: + dokku: + image: dokku/dokku:0.32.4 + container_name: dokku + network_mode: bridge + ports: + - "3022:22" + - "8080:80" + - "8443:443" + - "7000:7000" + volumes: + - "/var/lib/dokku:/mnt/dokku" + - "/var/run/docker.sock:/var/run/docker.sock" + environment: + DOKKU_HOSTNAME: dokku.me + DOKKU_HOST_ROOT: /var/lib/dokku/home/dokku + DOKKU_LIB_HOST_ROOT: /var/lib/dokku/var/lib/dokku + restart: unless-stopped + otel-collector: + image: otel/opentelemetry-collector:latest + container_name: otel-collector + network_mode: bridge + ports: + - "4318:4318" diff --git a/docker/docker-compose.test.yaml b/docker/docker-compose.test.yaml new file mode 100644 index 000000000..5e834102e --- /dev/null +++ b/docker/docker-compose.test.yaml @@ -0,0 +1,17 @@ +# This Docker Compose file is used specifically for running smoke tests on an alternative port +# to avoid conflicts with the Dokku deployment which uses port 7000. + +services: + prod-test: + image: opencodelists + container_name: docker-prod-test-1 + environment: + - SECRET_KEY=12345 + - DATABASE_DIR=/storage + - DATABASE_URL=sqlite:////storage/db.sqlite3 + - TRUD_API_KEY=dummy-key + - BASE_URLS=http://localhost:${PORT:-7001} + ports: + - "${PORT:-7001}:7000" # Map container's 7000 port to host's PORT (default 7001) + volumes: + - "./storage-test:/storage" diff --git a/docker/justfile b/docker/justfile index 6579e56f5..20457a634 100644 --- a/docker/justfile +++ b/docker/justfile @@ -69,11 +69,43 @@ _create_storage env: fi fi + if [ "{{ env }}" = "prod-test" ]; then + SCRIPT_DIR={{justfile_directory()}} + # Remove directory if it exists, then create fresh + echo 'Checking for storage-test directory...' + if test -d "$SCRIPT_DIR/storage-test"; then + echo 'Removing existing storage-test directory...' + sudo rm -rf "$SCRIPT_DIR/storage-test" + fi + echo 'Creating new storage-test directory...' + mkdir -p "$SCRIPT_DIR/storage-test" + echo 'Creating database file...' + touch "$SCRIPT_DIR/storage-test/db.sqlite3" + echo 'Setting ownership for storage-test directory...' + sudo chown 10003:10003 "$SCRIPT_DIR/storage-test" + echo 'Setting ownership for database file...' + sudo chown 10003:10003 "$SCRIPT_DIR/storage-test/db.sqlite3" + echo 'Storage test directory and database file created successfully.' + fi + # run dev server in docker container serve env="dev" *args="": (_create_storage env) docker compose up {{ args }} {{ env }} +# run test server on a different port (default 7001) +serve-test port="7001": (stop-test) (_create_storage "prod-test") + #!/usr/bin/env bash + set -euo pipefail + # Set environment variable for Docker Compose + export PORT={{ port }} + # Start the service with both compose files + docker compose -f {{ justfile_directory() }}/docker-compose.yaml -f {{ justfile_directory() }}/docker-compose.test.yaml up -d prod-test + + +# stop the test server +stop-test: + docker compose -f {{ justfile_directory() }}/docker-compose.yaml -f {{ justfile_directory() }}/docker-compose.test.yaml down prod-test # run command in dev container run env="dev" *args="bash": @@ -86,7 +118,7 @@ exec env="dev" *args="bash": # run a basic functional smoke test against a running opencodelists -smoke-test host="http://localhost:7000": +smoke-test host="http://localhost:7000" env="prod": #!/bin/bash set -eu curl -I {{ host }} -s --compressed --fail --retry 20 --retry-delay 1 --retry-all-errors diff --git a/justfile b/justfile index bcae3e0b5..94aaa8a7e 100644 --- a/justfile +++ b/justfile @@ -330,6 +330,14 @@ docker-test: _env docker-serve env="dev" *args="": _env {{ just_executable() }} docker/serve {{ if env == "dev" { docker_env } else { env } }} {{ args }} +# run test server on a different port +docker-serve-test port="7001": _env + {{ just_executable() }} docker/serve-test {{ port }} + +# stop the test server +docker-stop-test: _env + {{ just_executable() }} docker/stop-test + # run cmd in dev docker continer docker-run *args="bash": _env @@ -342,8 +350,8 @@ docker-exec *args="bash": _env # run tests in docker container -docker-smoke-test host="http://localhost:7000": _env - {{ just_executable() }} docker/smoke-test {{ host }} +docker-smoke-test host="http://localhost:7000" env="prod": _env + {{ just_executable() }} docker/smoke-test {{ host }} {{env}} # check migrations in the dev docker container diff --git a/local-ci-pipeline.sh b/local-ci-pipeline.sh new file mode 100755 index 000000000..f05896340 --- /dev/null +++ b/local-ci-pipeline.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -eu + +# This script replicates the CI build and test pipeline locally. It runs the +# same checks and tests as the GitHub Actions workflows in main.yml + +# Function to clean up containers when done +cleanup() { + echo "> Cleaning up smoke test containers..." + just docker-stop-test + + echo "Ensure staticfiles directory exists and the .keep file hasn't been overwritten" + mkdir -p docker/docker-staticfiles + touch docker/docker-staticfiles/.keep +} + +# Set up trap to ensure cleanup happens even if script errors out +trap cleanup EXIT + +echo "> Running jobs from main.yml locally..." + +echo "> Simulating job 'check-py' from main.yml" +just docker-check-py + +echo "> Simulating job 'check-js' from main.yml" +just docker-check-js + +echo "> Simulating job 'test-py' from main.yml" +# Step 1: Build images for both prod and dev +just docker-build prod +just docker-build dev + +# Step 2: Check migrations +just docker-check-migrations + +# Step 3: Run unit tests +just docker-test-py --migrations + +# Step 4: Run smoke test +# The command in main.yml is +# just docker-serve prod -d +# But that won't work because port 7000 is already in use by the locally +# running dokku instance. +# So we run the test server on port 7001 instead with a slightly different just +# command. +just docker-serve-test 7001 +sleep 5 +just docker-smoke-test http://localhost:7001 prod-test || { docker logs docker-prod-test-1; echo "❌ Smoke test failed."; exit 1; } + +# Step 5: Save docker image +# We currently don't test this step locally +# Step 6: Upload docker image +# This is not applicable for local testing, so we skip it + +echo "> Simulating job 'test-js' from main.yml" +just docker-test-js + +# If we got here, all tests passed +echo "✅ All CI tests passed locally!" diff --git a/local-deploy.sh b/local-deploy.sh new file mode 100755 index 000000000..522328768 --- /dev/null +++ b/local-deploy.sh @@ -0,0 +1,65 @@ +#!/bin/bash +set -eu + +WORKSPACE="${containerWorkspaceFolder:?containerWorkspaceFolder not set}" +echo "Using workspace: $WORKSPACE" + +# INSTALL JUST +echo "Installing just command runner..." +cd /tmp && wget https://github.com/casey/just/releases/download/1.40.0/just-1.40.0-x86_64-unknown-linux-musl.tar.gz && tar -xzf just-1.40.0-x86_64-unknown-linux-musl.tar.gz && chmod 555 just +sudo mv /tmp/just /usr/bin/ +echo "✓ Just installed successfully" + +cd "$WORKSPACE" +touch db.sqlite3 + +echo "Creating storage directories..." +sudo mkdir -p /var/lib/dokku/data/storage/opencodelists +sudo mkdir -p /var/lib/dokku/data/storage/opencodelists/coding_systems/bnf/ +sudo mkdir -p /var/lib/dokku/data/storage/opencodelists/coding_systems/dmd/ +sudo mkdir -p /var/lib/dokku/data/storage/opencodelists/coding_systems/icd10/ +sudo mkdir -p /var/lib/dokku/data/storage/opencodelists/coding_systems/snomedct/ + +echo "Copying database..." +sudo cp db.sqlite3 /var/lib/dokku/data/storage/opencodelists/ + +echo "Setting permissions..." +sudo chown -R 10003:10003 /var/lib/dokku/data/storage/opencodelists + +# Wait for Docker daemon to be available +max_attempts=30 +attempt=1 +while ! docker info >/dev/null 2>&1; do + if [ "$attempt" -ge "$max_attempts" ]; then + echo "Docker daemon did not start after $max_attempts attempts. Exiting." + exit 1 + fi + echo "Waiting for Docker daemon to start... ($attempt/$max_attempts)" + sleep 2 + attempt=$((attempt+1)) +done + +# Show Docker system info +docker info + +if ! docker compose up -d; then + echo "Docker Compose failed. Trying again after delay..." + sleep 10 + docker compose up -d || { + echo "Docker Compose failed again. See errors above." + exit 1 + } +fi + +# Get otel-collector container IP address +otel_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' otel-collector) + +just docker-build prod +docker tag opencodelists dokku/opencodelists:latest + +docker exec -t dokku sh -c "dokku apps:destroy opencodelists --force" || true +docker exec -t dokku sh -c "dokku apps:create opencodelists" +docker exec -t dokku sh -c "dokku storage:mount opencodelists /var/lib/dokku/data/storage/opencodelists/:/storage" +docker exec -t dokku sh -c "dokku config:set opencodelists BASE_URLS=\"http://localhost:7000,http://127.0.0.1:7000\" DATABASE_DIR=\"/storage\" DATABASE_URL=\"sqlite:////storage/db.sqlite3\" DJANGO_SETTINGS_MODULE=\"opencodelists.settings\" SECRET_KEY=\"thisisatestsecretkeyfortestingonly\" TRUD_API_KEY=\"thisisatesttrudkeyfortestingonly\" OTEL_EXPORTER_OTLP_ENDPOINT=\"http://$otel_ip:4318\"" +docker exec -t dokku sh -c "dokku git:from-image opencodelists dokku/opencodelists:latest" +docker image prune -a -f diff --git a/local-redeploy.sh b/local-redeploy.sh new file mode 100755 index 000000000..86fded2c7 --- /dev/null +++ b/local-redeploy.sh @@ -0,0 +1,100 @@ +#!/bin/bash +set -eu + +# Default to running a minimal deployment without full CI pipeline +RUN_CI=${RUN_CI:-0} +# Accept target branch as parameter, default to current branch +TARGET_BRANCH=${1:-$(git rev-parse --abbrev-ref HEAD)} + +# Store original directory +ORIGINAL_DIR="$PWD" + +# Function to clean up when done +cleanup() { + echo "Ensure staticfiles directory exists and the .keep file hasn't been overwritten" + mkdir -p docker/docker-staticfiles + touch docker/docker-staticfiles/.keep + + # If we created a temp dir, clean it up + if [ -n "${TEMP_DIR:-}" ] && [ -d "$TEMP_DIR" ]; then + echo "Cleaning up temporary directory..." + rm -rf "$TEMP_DIR" + fi + + # Return to original directory + cd "$ORIGINAL_DIR" +} + +# Set up trap to ensure cleanup happens even if script errors out +trap cleanup EXIT + +# If testing a different branch than current, set up temp directory +if [ "$TARGET_BRANCH" != "$(git rev-parse --abbrev-ref HEAD)" ]; then + echo "> Testing deployment for branch: $TARGET_BRANCH" + + # Make sure we have all remote branches fetched in the original repository + echo "> Fetching all remote branches..." + cd "$ORIGINAL_DIR" + git fetch --all --prune + + # Create a local branch from the remote TARGET_BRANCH if it doesn't exist locally + if ! git rev-parse --verify --quiet "$TARGET_BRANCH" >/dev/null; then + echo "> Creating local copy of remote branch: $TARGET_BRANCH" + git branch "$TARGET_BRANCH" "origin/$TARGET_BRANCH" + fi + + # Verify we now have the branch locally + if ! git rev-parse --verify --quiet "$TARGET_BRANCH" >/dev/null; then + echo "❌ Could not find or create branch: $TARGET_BRANCH" + exit 1 + fi + + # Create a temporary directory and clone the repository + TEMP_DIR=$(mktemp -d) + echo "> Creating temporary workspace at $TEMP_DIR" + + # Clone the repository into the temp dir (shallow clone for speed) + git clone "$PWD" "$TEMP_DIR" + cd "$TEMP_DIR" + + # Fetch the target branch and check it out + echo "> Checking out branch: $TARGET_BRANCH" + git fetch origin "$TARGET_BRANCH":refs/remotes/origin/"$TARGET_BRANCH" --depth 1 + git checkout -b "$TARGET_BRANCH" origin/"$TARGET_BRANCH" || { + echo "❌ Failed to checkout branch $TARGET_BRANCH" + exit 1 + } + + echo "✅ Now working with code from branch: $TARGET_BRANCH" +else + echo "> Using current branch: $TARGET_BRANCH" +fi + +if [ "$RUN_CI" = "1" ]; then + echo "> Running full CI pipeline tests before deployment..." + ./local-ci-pipeline.sh + echo "> CI pipeline tests passed. Proceeding with deployment..." +else + echo "> Skipping CI pipeline tests. Use RUN_CI=1 to run them before deployment." + echo "> Building new Docker image with latest changes..." + just docker-build prod +fi + +# Tag the built prod image for local dokku deployment +docker tag opencodelists dokku/opencodelists:latest + +echo "> Deploying to Dokku..." +# Use || true to ensure script continues even if git:from-image exits with non-zero status +docker exec -t dokku sh -c "dokku git:from-image opencodelists dokku/opencodelists:latest" || true +# Always run ps:rebuild to ensure changes like Procfile updates are applied +echo "> Rebuilding app processes..." +docker exec -t dokku sh -c "dokku ps:rebuild opencodelists" + +echo "> Cleaning up old images..." +docker exec -t dokku sh -c "dokku releases:purge opencodelists --keep 2" || true +docker image prune -a -f + +echo "✅ Deployment complete!" +echo " Branch tested: $TARGET_BRANCH" +echo " Check app at: http://localhost:7000" +echo " View logs with: docker exec -t dokku sh -c \"dokku logs opencodelists -t\""