diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 38f0f55..cc653d8 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -73,6 +73,8 @@ jobs: - name: Comment on PR if: always() + env: + PREVIEW_DOMAIN: ${{ vars.PREVIEW_DOMAIN }} uses: actions/github-script@v7 with: github-token: ${{ steps.app-token.outputs.token || github.token }} @@ -83,7 +85,7 @@ jobs: const sha = context.payload.pull_request.head.sha; const shortSha = sha.slice(0, 7); const branch = context.payload.pull_request.head.ref; - const url = `https://pr-${pr}.preview.lyrics-api.boidu.dev`; + const url = `https://pr-${pr}.${process.env.PREVIEW_DOMAIN}`; const runUrl = `${context.serverUrl}/${owner}/${repo}/actions/runs/${context.runId}`; const commitUrl = `${context.serverUrl}/${owner}/${repo}/commit/${sha}`; const now = new Date().toISOString().replace('T', ' ').slice(0, 19) + ' UTC'; diff --git a/README.md b/README.md index dbb0de6..6e963c3 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@ ![GitHub top language](https://img.shields.io/github/languages/top/better-lyrics/api) ![GitHub License](https://img.shields.io/github/license/better-lyrics/api) ![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/better-lyrics/api/go.yml) -![Railway](https://img.shields.io/badge/deployment-railway-javascript?logo=railway&logoColor=fff&color=851AE6) This repository contains the source code for the official Better Lyrics API - primarily serving as the backend for [Better Lyrics](https://better-lyrics.boidu.dev). @@ -12,69 +11,49 @@ This repository contains the source code for the official Better Lyrics API - pr ## Table of Contents -- [Better Lyrics API](#better-lyrics-api) - - [Table of Contents](#table-of-contents) - - [Installation](#installation) - - [Usage](#usage) - - [API Endpoints](#api-endpoints) - - [Deployment](#deployment) - - [Railway](#railway) - - [Contributing](#contributing) - - [License](#license) +- [Quickstart](#quickstart) +- [API Endpoints](#api-endpoints) +- [Deployment](#deployment) +- [Contributing](#contributing) +- [License](#license) -## Installation +## Quickstart -To install and run the Lyrics API Go, follow these steps: +If you just want to run it locally, you need a Go toolchain (1.22+) and a populated `.env`. -1. Clone the repository: `git clone https://github.com/better-lyrics/api.git` -2. Navigate to the project directory: `cd api` -3. Install the dependencies: `go mod tidy` -4. Copy the `.env.example` file to `.env` and update the environment variables as needed: `cp .env.example .env` -5. Start the server: `go run main.go` - -## Usage +```bash +git clone https://github.com/better-lyrics/api.git && cd api +go mod tidy +cp .env.example .env # fill in upstream API endpoints + credentials +go run main.go # serves on :8080 +``` -Once the server is running, you can access the API endpoints to retrieve lyrics for songs. +The server logs request lines as it boots; once you see the listener line, hit `http://localhost:8080/health`. For hot reload during development, `./scripts/run.sh` watches the source via `nodemon`. ## API Endpoints -- `GET /getLyrics?a={artist}&s={song}`: Retrieves the lyrics for the specified artist and song. - -## Deployment - -### Railway - -This project uses Railway's persistent volumes to maintain the cache database across deployments. +Public: -**Setup Steps:** +- `GET /getLyrics?a={artist}&s={song}` - Retrieves synchronized lyrics for the specified artist and song +- `GET /artwork?s={song}&a={artist}` - Returns animated album artwork +- `GET /health` - Health check +- `GET /stats` - API statistics (requires `Authorization` header) -1. Create a new project on [Railway](https://railway.app) and connect your GitHub repository +A full list of admin/cache endpoints (`/cache/*`, `/revalidate`, `/override`, `/health/mut`, etc.) is documented in [`CLAUDE.md`](./CLAUDE.md). -2. **Create a Volume (CRITICAL):** - - Go to your service in Railway dashboard - - Click **Settings** → **Volumes** tab - - Click **+ New Volume** - - **Mount Path:** `/data` - - Click **Add** +## Deployment -3. **Set Environment Variables:** - - Go to **Variables** tab - - Add: `CACHE_DB_PATH=/data/cache.db` - - Configure all other required variables from `.env.example` +Production runs on a single Hetzner CAX21 (ARM64, Helsinki). The whole server stack (Caddy, the API, Infisical agent for secrets sync, Beszel agent for metrics, Logdy for log streaming, B2 backups, UFW, fail2ban) lives in [`infra/`](./infra/README.md) as code. -4. **Deploy!** +To rebuild from scratch on any Ubuntu 24.04 host: -**Verification:** -After deployment, check your logs for: -``` -[Cache] Loaded X entries from disk to memory +```bash +cp infra/secrets.env.example infra/secrets.env +$EDITOR infra/secrets.env # fill in every value from your password manager +sudo ./infra/bootstrap.sh # about 10 minutes, idempotent ``` -If you see `Loaded 0 entries` on subsequent deploys (after caching data), the volume isn't persisting. -**Troubleshooting:** -- Ensure the volume mount path is exactly `/data` -- Verify `CACHE_DB_PATH=/data/cache.db` is set in Railway variables -- The volume must be created BEFORE deploying with the updated env var +See [`infra/README.md`](./infra/README.md) for the prerequisites and the manual steps that stay manual (DNS, provisioning, `cache.db` restore). ## Contributing diff --git a/infra/.gitignore b/infra/.gitignore new file mode 100644 index 0000000..5a3eb42 --- /dev/null +++ b/infra/.gitignore @@ -0,0 +1,2 @@ +secrets.env +*.bak diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 0000000..0cfb54c --- /dev/null +++ b/infra/README.md @@ -0,0 +1,127 @@ +# infra/ + +Everything needed to stand up the server lives here. If the Hetzner box dies or you want to move providers, `sudo ./bootstrap.sh` on a fresh Ubuntu 24.04 host rebuilds it in about ten minutes. + +## What runs on the box + +| Component | Purpose | Reachable at | +|---|---|---| +| `caddy` | Reverse proxy with TLS via Cloudflare DNS-01 | 80/443 | +| `lyrics-api` | The Go API | localhost:8080 (proxied) | +| `lyrics-api@.service` | Per-PR preview environments | localhost:9000+PR | +| `infisical-agent` | Syncs prod secrets from Infisical Cloud into `/etc/lyrics-api.env` and restarts the API on change | n/a | +| `beszel-agent` | System metrics, reports to a hub | localhost:45876 | +| `logdy` | Browser log viewer for `lyrics-api` journal | localhost:8888 (proxied) | +| Backup scripts | Daily `cache.db` dump, off-site upload to Backblaze B2 | cron | +| `ufw` + `fail2ban` | Firewall + sshd brute-force protection | n/a | + +## Prerequisites + +On the box: +- Fresh Ubuntu 24.04 (or compatible) with sudo and outbound network +- SSH access for the account that will run `sudo` + +Off the box, before you run bootstrap: +- DNS records pre-pointed at the box (see "Manual steps" below) +- A Cloudflare API token scoped to Zone:DNS:Edit on the parent zone of your domains +- A Backblaze B2 application key with read+write on the backups bucket +- An Infisical Cloud machine identity (Universal Auth) with read access to the `prod` env +- A Beszel hub somewhere reachable, with an agent KEY/TOKEN pair generated for this host +- `secrets.env` populated next to `bootstrap.sh` (template: `secrets.env.example`) + +The compiled `lyrics-api-go` binary is optional at bootstrap time. Phase 04 installs the systemd unit either way and waits to start it until both the binary and `/etc/lyrics-api.env` exist. + +## Running + +```bash +# from the repo root on the target box +cp infra/secrets.env.example infra/secrets.env +$EDITOR infra/secrets.env # fill every blank +sudo ./infra/bootstrap.sh # full bootstrap +sudo ./infra/bootstrap.sh --phase 03 # re-run a single phase (here: Caddy) +``` + +Logs go to `/var/log/bli-bootstrap.log`. Phases are idempotent: re-running reconciles drift (rewriting systemd units to match repo state, refreshing config files) without breaking anything already in place. + +## Manual steps (not automated) + +These happen outside the box and stay manual: + +- **Provision the Hetzner instance** with `hcloud server create --type cax21 --image ubuntu-24.04 --location hel1 ...` +- **DNS records in Cloudflare** for the four hostnames in `secrets.env` plus the preview wildcard, all proxied (orange cloud) +- **Infisical secrets** in the project's `prod` env. The agent only syncs; it does not create. +- **Beszel hub** running somewhere reachable, with an agent slot for this host. The hub UI hands you the KEY/TOKEN pair for `secrets.env`. +- **`cache.db` restore** from B2 if you're rebuilding after a loss. Separate process: `rclone copy b2:lyrics-api-backups/daily/ /var/lib/lyrics-api/data/cache.db`. + +## Deploying the lyrics-api binary + +The IaC installs the systemd unit but does not ship the Go binary. Two ways to get it on the box: + +1. **From CI** (the path used in prod): GitHub Actions builds, `scp`s to `/opt/lyrics-api/lyrics-api-go`, then `systemctl restart lyrics-api`. +2. **From source on the box**: `git clone`, `go build -o /opt/lyrics-api/lyrics-api-go .`, `chown deploy:deploy`, `systemctl restart lyrics-api`. + +Either way, `infisical-agent` writes `/etc/lyrics-api.env` once it starts, which is what unblocks the first `lyrics-api` start. + +## Security model + +Most secrets live in Infisical Cloud and sync read-only to the box. The exceptions, all kept off the world-readable systemd config: + +- `CF_API_TOKEN` is in `/etc/caddy.env` (mode 600, root:caddy) +- `B2_*` is in `/home/deploy/.config/rclone/rclone.conf` (mode 600, deploy:deploy) +- `LOGDY_UI_PASS` is in `/etc/logdy.env` (mode 640, root:deploy) +- The Infisical `client-secret` is at `/etc/infisical-agent/client-secret` (mode 600, root) + +`BESZEL_AGENT_TOKEN` is the one wart: it sits in `Environment=` lines on a mode-644 unit, since that's how the upstream installer ships it. The blast radius if leaked is impersonating the agent to the hub, which sends fake metrics but does not grant credentials back. The same EnvironmentFile pattern Caddy uses would close it; it is not done yet. + +`lyrics-api.service` itself runs as `deploy` with `ProtectSystem=strict`, `ProtectHome=true`, `PrivateTmp=true`, `NoNewPrivileges=true`. UFW restricts inbound traffic to 22/80/443. `fail2ban` watches `sshd`. + +## Verification after bootstrap + +Substitute your own hostnames from `secrets.env` for `$PRIMARY_DOMAIN` and `$LOGS_DOMAIN`. + +```bash +systemctl is-active caddy lyrics-api infisical-agent beszel-agent logdy fail2ban +ls -l /etc/caddy.env # -rw------- root caddy +sudo -u nobody cat /etc/caddy.env # permission denied +curl -sI https://$PRIMARY_DOMAIN/health # 200 +curl -sI https://$LOGS_DOMAIN/ # 200 (then 401 on actual UI without auth) +journalctl -u infisical-agent -n 20 # successful auth + sync +``` + +## Selective rebuild scenarios + +| Scenario | Command | +|---|---| +| Caddy config changed | `sudo ./bootstrap.sh --phase 03` | +| Memory drop-in needs adjusting | edit `files/lyrics-api/memory.conf`, then `sudo ./bootstrap.sh --phase 04` | +| Backup schedule changed | edit `files/backups/crontab.fragment`, then `sudo ./bootstrap.sh --phase 08` | +| Logdy version bump | bump `LOGDY_VERSION` in `secrets.env`, then `sudo ./bootstrap.sh --phase 07` | +| Beszel agent token rotated | update `BESZEL_AGENT_TOKEN` in `secrets.env`, then `sudo ./bootstrap.sh --phase 06` | + +## What's intentionally not here + +- **Provisioning** (`hcloud server create`). One command, varies per provider, not worth scripting. +- **DNS records**. Lives in Cloudflare; the UI is fine. +- **The `lyrics-api-go` binary**. Shipped from CI, not infra. +- **A self-hosted Infisical instance**. 600MB resident is more than the project warrants. +- **`cache.db` restoration**. Separate runbook, depends on which B2 snapshot you want. + +## GitHub Actions configuration + +Two workflows talk to this box: `.github/workflows/deploy-hetzner.yml` (prod deploys on push to `master`) and `.github/workflows/preview.yml` (per-PR previews). Both need configuration set in the GitHub repo: + +Repository **secrets** (Settings > Secrets and variables > Actions > Secrets): +- `APP_ID`, `APP_PRIVATE_KEY` for the GitHub App that posts PR comments +- `SSH_KEY` private key authorised on the box for the deploy user +- `SSH_HOST` the box's IP or DNS name +- `SSH_USER` the SSH login (e.g. `deploy`) + +Repository **variables** (Settings > Secrets and variables > Actions > Variables): +- `PREVIEW_DOMAIN` the wildcard zone for PR previews, e.g. `preview.api.example.com`. Must match `PREVIEW_WILDCARD` in `secrets.env` on the box (same value, minus the leading `*.`). + +## When something breaks + +1. Read `/var/log/bli-bootstrap.log` for the failing phase +2. Re-run just that phase with `--phase NN` +3. If the failure is upstream (apt repo down, a GitHub release moved), the phase script is the source of truth. Open it, fix it, re-run. +4. For infisical-agent issues, `journalctl -t infisical-agent -n 50` shows the reload script's logger output. diff --git a/infra/bootstrap.sh b/infra/bootstrap.sh new file mode 100755 index 0000000..eaf7cbd --- /dev/null +++ b/infra/bootstrap.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Better Lyrics API - bootstrap a fresh Ubuntu 24.04 box from zero to production. +# Usage: sudo ./bootstrap.sh [--phase NN] +# With --phase NN, runs only that single phase (e.g. --phase 03 to reconfigure Caddy). +# Source: secrets.env (must exist next to this script). +# Logs: /var/log/bli-bootstrap.log + +set -euo pipefail + +INFRA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LOG=/var/log/bli-bootstrap.log +SECRETS="${INFRA_DIR}/secrets.env" + +if [ "$EUID" -ne 0 ]; then + echo "ERROR: bootstrap.sh must run as root (use sudo)" >&2 + exit 1 +fi + +if [ ! -f "$SECRETS" ]; then + echo "ERROR: secrets.env not found at $SECRETS" >&2 + echo "Copy secrets.env.example to secrets.env and fill in values." >&2 + exit 1 +fi + +# shellcheck disable=SC1090 +set -a; source "$SECRETS"; set +a + +REQUIRED_VARS=( + CF_API_TOKEN ACME_EMAIL + INFISICAL_CLIENT_ID INFISICAL_CLIENT_SECRET INFISICAL_PROJECT_ID INFISICAL_ENV + B2_KEY_ID B2_APP_KEY B2_BUCKET + BESZEL_HUB_URL BESZEL_AGENT_KEY BESZEL_AGENT_TOKEN BESZEL_AGENT_PORT + LOGDY_UI_PASS LOGDY_VERSION + PUBLIC_IP PRIMARY_DOMAIN STAGING_DOMAIN LOGS_DOMAIN METRICS_DOMAIN PREVIEW_WILDCARD +) +missing=() +for v in "${REQUIRED_VARS[@]}"; do + if [ -z "${!v:-}" ]; then missing+=("$v"); fi +done +if [ ${#missing[@]} -gt 0 ]; then + echo "ERROR: missing required secrets: ${missing[*]}" >&2 + exit 1 +fi + +export INFRA_DIR + +mkdir -p "$(dirname "$LOG")" +exec > >(tee -a "$LOG") 2>&1 + +ONLY_PHASE="" +if [ "${1:-}" = "--phase" ]; then + ONLY_PHASE="${2:?--phase requires a number}" +fi + +run_phase() { + local script="$1" + local name + name=$(basename "$script") + echo + echo "===> $name $(date -u +%Y-%m-%dT%H:%M:%SZ)" + bash "$script" + echo "<=== $name OK" +} + +echo "=== Better Lyrics API bootstrap ===" +echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)" +echo "Host: $(hostname)" +echo "Infra dir: $INFRA_DIR" +echo + +for script in "$INFRA_DIR"/phases/[0-9][0-9]-*.sh; do + if [ -n "$ONLY_PHASE" ]; then + case "$(basename "$script")" in + "${ONLY_PHASE}-"*) run_phase "$script" ;; + esac + else + run_phase "$script" + fi +done + +echo +echo "=== bootstrap complete ===" +echo "Verify with:" +echo " systemctl is-active caddy lyrics-api infisical-agent beszel-agent logdy fail2ban" +echo " curl -sI https://${PRIMARY_DOMAIN}/health" diff --git a/infra/files/backups/backup-cache.sh b/infra/files/backups/backup-cache.sh new file mode 100755 index 0000000..9020b9b --- /dev/null +++ b/infra/files/backups/backup-cache.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -euo pipefail +source /etc/lyrics-api.env +curl -s -H "Authorization: $CACHE_ACCESS_TOKEN" http://localhost:8080/cache/backup > /dev/null diff --git a/infra/files/backups/crontab.fragment b/infra/files/backups/crontab.fragment new file mode 100644 index 0000000..8ebb053 --- /dev/null +++ b/infra/files/backups/crontab.fragment @@ -0,0 +1,5 @@ +# deploy user crontab - managed by infra/phases/08-backups.sh +# Backup local cache.db at 03:00 UTC, upload to B2 at 03:30 UTC, prune local at 04:00 UTC +0 3 * * * /usr/local/bin/backup-cache.sh +30 3 * * * /usr/local/bin/upload-backup.sh +0 4 * * * find /var/lib/lyrics-api/backups -name 'cache_backup_*.db' -mtime +1 -delete diff --git a/infra/files/backups/upload-backup.sh b/infra/files/backups/upload-backup.sh new file mode 100755 index 0000000..39fb0d5 --- /dev/null +++ b/infra/files/backups/upload-backup.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail + +LATEST=$(ls -1t /var/lib/lyrics-api/backups/cache_backup_*.db 2>/dev/null | head -1) +[ -z "$LATEST" ] && { echo "No backup file found"; exit 1; } + +# Always upload to daily/ +/usr/bin/rclone copy "$LATEST" b2:lyrics-api-backups/daily/ --transfers 4 + +# On Sundays only, also upload to weekly/ +if [ "$(date -u +%u)" = "7" ]; then + /usr/bin/rclone copy "$LATEST" b2:lyrics-api-backups/weekly/ --transfers 4 +fi diff --git a/infra/files/caddy/Caddyfile b/infra/files/caddy/Caddyfile new file mode 100644 index 0000000..952bfe6 --- /dev/null +++ b/infra/files/caddy/Caddyfile @@ -0,0 +1,60 @@ +# Hostname placeholders are substituted from secrets.env at install time by phases/03-caddy.sh. +# If you're forking this, populate PRIMARY_DOMAIN, STAGING_DOMAIN, etc. in your own secrets.env. + +{ + email __ACME_EMAIL__ +} + +(security_headers) { + header { + Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" + X-Content-Type-Options "nosniff" + Referrer-Policy "strict-origin-when-cross-origin" + -Server + } +} + +__STAGING_DOMAIN__ { + reverse_proxy localhost:8080 + encode zstd gzip + tls { + dns cloudflare {env.CF_API_TOKEN} + } + import security_headers +} + +__PRIMARY_DOMAIN__ { + tls { + dns cloudflare {env.CF_API_TOKEN} + } + import security_headers + reverse_proxy localhost:8080 + encode zstd gzip +} + +__METRICS_DOMAIN__ { + tls { + dns cloudflare {env.CF_API_TOKEN} + } + import security_headers + reverse_proxy localhost:8090 +} + +__LOGS_DOMAIN__ { + tls { + dns cloudflare {env.CF_API_TOKEN} + } + import security_headers + reverse_proxy localhost:8888 +} + +__PREVIEW_WILDCARD__ { + tls { + dns cloudflare {env.CF_API_TOKEN} + } + import security_headers + import /etc/caddy/previews/*.caddy + handle { + respond "Preview not found or expired" 404 + } +} diff --git a/infra/files/caddy/caddy.env.example b/infra/files/caddy/caddy.env.example new file mode 100644 index 0000000..951a1eb --- /dev/null +++ b/infra/files/caddy/caddy.env.example @@ -0,0 +1,4 @@ +# Caddy environment file - populated by bootstrap from secrets.env +# Mode: 600, owner: root:caddy +# Caddy reads this via systemd EnvironmentFile= directive +CF_API_TOKEN= diff --git a/infra/files/caddy/caddy.service.d/override.conf b/infra/files/caddy/caddy.service.d/override.conf new file mode 100644 index 0000000..791c7d5 --- /dev/null +++ b/infra/files/caddy/caddy.service.d/override.conf @@ -0,0 +1,2 @@ +[Service] +EnvironmentFile=/etc/caddy.env diff --git a/infra/files/infisical/agent.yaml b/infra/files/infisical/agent.yaml new file mode 100644 index 0000000..f6390a6 --- /dev/null +++ b/infra/files/infisical/agent.yaml @@ -0,0 +1,23 @@ +infisical: + address: "https://app.infisical.com" + +auth: + type: "universal-auth" + config: + client-id: "/etc/infisical-agent/client-id" + client-secret: "/etc/infisical-agent/client-secret" + remove_client_secret_on_read: false + +sinks: + - type: "file" + config: + path: "/etc/infisical-agent/access-token" + +templates: + - source-path: "/etc/infisical-agent/lyrics-api.env.tpl" + destination-path: "/etc/lyrics-api.env.staging" + config: + polling-interval: 60s + execute: + timeout: 30 + command: "/usr/local/bin/lyrics-api-reload" diff --git a/infra/files/infisical/infisical-agent.service b/infra/files/infisical/infisical-agent.service new file mode 100644 index 0000000..774ea9d --- /dev/null +++ b/infra/files/infisical/infisical-agent.service @@ -0,0 +1,17 @@ +[Unit] +Description=Infisical Agent (secrets sync) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=root +ExecStart=/usr/bin/infisical agent --config /etc/infisical-agent/agent.yaml +Restart=on-failure +RestartSec=10 +MemoryMax=256M +NoNewPrivileges=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/infra/files/infisical/lyrics-api-reload b/infra/files/infisical/lyrics-api-reload new file mode 100755 index 0000000..a84495c --- /dev/null +++ b/infra/files/infisical/lyrics-api-reload @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +STAGING="/etc/lyrics-api.env.staging" +REAL="/etc/lyrics-api.env" + +if [ ! -s "$STAGING" ]; then + logger -t infisical-agent "ERROR: staged env is empty, aborting" + exit 1 +fi + +if ! grep -q '^TTML_MEDIA_USER_TOKENS=' "$STAGING"; then + logger -t infisical-agent "ERROR: TTML_MEDIA_USER_TOKENS missing in staged env, aborting" + exit 1 +fi + +if [ -f "$REAL" ] && cmp -s "$STAGING" "$REAL"; then + logger -t infisical-agent "no change in env, skipping restart" + rm -f "$STAGING" + exit 0 +fi + +[ -f "$REAL" ] && cp "$REAL" "${REAL}.bak" +mv "$STAGING" "$REAL" +chmod 640 "$REAL" +chown root:deploy "$REAL" + +systemctl restart lyrics-api +logger -t infisical-agent "lyrics-api restarted after secret change" diff --git a/infra/files/infisical/lyrics-api.env.tpl b/infra/files/infisical/lyrics-api.env.tpl new file mode 100644 index 0000000..d505d2c --- /dev/null +++ b/infra/files/infisical/lyrics-api.env.tpl @@ -0,0 +1,5 @@ +{{- with listSecrets "__INFISICAL_PROJECT_ID__" "__INFISICAL_ENV__" "/" }} +{{- range . }} +{{ .Key }}={{ .Value }} +{{- end }} +{{- end }} diff --git a/infra/files/logdy/logdy.env.example b/infra/files/logdy/logdy.env.example new file mode 100644 index 0000000..c8f45bc --- /dev/null +++ b/infra/files/logdy/logdy.env.example @@ -0,0 +1,4 @@ +# Logdy environment file - populated by bootstrap from secrets.env +# Mode: 640, owner: root:deploy +# Logdy reads LOGDY_UI_PASS at startup; username is logdy's built-in default +LOGDY_UI_PASS= diff --git a/infra/files/logdy/logdy.service b/infra/files/logdy/logdy.service new file mode 100644 index 0000000..32005b7 --- /dev/null +++ b/infra/files/logdy/logdy.service @@ -0,0 +1,18 @@ +[Unit] +Description=Logdy log viewer for lyrics-api +After=network.target + +[Service] +Type=simple +User=deploy +Group=systemd-journal +EnvironmentFile=/etc/logdy.env +ExecStart=/bin/bash -c '/usr/bin/journalctl -f -u lyrics-api -o cat -n 1000 | /usr/local/bin/logdy --port 8888 --ui-ip 127.0.0.1 --disable-ansi-code-stripping --no-analytics --no-updates' +Restart=always +RestartSec=5 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true + +[Install] +WantedBy=multi-user.target diff --git a/infra/files/lyrics-api/lyrics-api.service b/infra/files/lyrics-api/lyrics-api.service new file mode 100644 index 0000000..138eb8c --- /dev/null +++ b/infra/files/lyrics-api/lyrics-api.service @@ -0,0 +1,22 @@ +[Unit] +Description=Better Lyrics API +After=network.target + +[Service] +Type=simple +User=deploy +Group=deploy +WorkingDirectory=/opt/lyrics-api +EnvironmentFile=/etc/lyrics-api.env +ExecStart=/opt/lyrics-api/lyrics-api-go +Restart=on-failure +RestartSec=2 +LimitNOFILE=65536 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/var/lib/lyrics-api + +[Install] +WantedBy=multi-user.target diff --git a/infra/files/lyrics-api/memory.conf b/infra/files/lyrics-api/memory.conf new file mode 100644 index 0000000..16ece5c --- /dev/null +++ b/infra/files/lyrics-api/memory.conf @@ -0,0 +1,3 @@ +[Service] +MemoryHigh=5G +MemoryMax=6G diff --git a/infra/files/lyrics-api/sudoers.deploy-preview b/infra/files/lyrics-api/sudoers.deploy-preview new file mode 100644 index 0000000..5275ee0 --- /dev/null +++ b/infra/files/lyrics-api/sudoers.deploy-preview @@ -0,0 +1,3 @@ +# Allow deploy user to run preview deploy/teardown scripts without password +# Used by .github/workflows/preview.yml to manage per-PR preview environments +deploy ALL=(root) NOPASSWD: /usr/local/bin/preview-deploy.sh, /usr/local/bin/preview-teardown.sh diff --git a/infra/files/rclone/rclone.conf.example b/infra/files/rclone/rclone.conf.example new file mode 100644 index 0000000..873c551 --- /dev/null +++ b/infra/files/rclone/rclone.conf.example @@ -0,0 +1,6 @@ +# rclone config for B2 - populated by bootstrap from secrets.env into /home/deploy/.config/rclone/rclone.conf +# Mode: 600, owner: deploy:deploy +[b2] +type = b2 +account = __B2_KEY_ID__ +key = __B2_APP_KEY__ diff --git a/infra/phases/00-prereqs.sh b/infra/phases/00-prereqs.sh new file mode 100755 index 0000000..ea9a338 --- /dev/null +++ b/infra/phases/00-prereqs.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -euo pipefail + +# Verify Ubuntu 24.04 (or compatible) and basic network +. /etc/os-release +case "$ID" in + ubuntu|debian) ;; + *) echo "WARN: untested distro $ID, proceeding anyway" ;; +esac + +if ! ping -c 1 -W 2 1.1.1.1 > /dev/null 2>&1; then + echo "ERROR: no network connectivity" + exit 1 +fi + +if ! command -v curl > /dev/null; then + apt-get update + apt-get install -y curl +fi + +echo "OK: $PRETTY_NAME, network reachable" diff --git a/infra/phases/01-packages.sh b/infra/phases/01-packages.sh new file mode 100755 index 0000000..2e8ea8b --- /dev/null +++ b/infra/phases/01-packages.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -euo pipefail + +# Base packages: build/network/security tooling +DEBIAN_FRONTEND=noninteractive apt-get update +DEBIAN_FRONTEND=noninteractive apt-get install -y \ + curl wget ca-certificates gnupg lsb-release \ + ufw fail2ban \ + rclone jq git \ + apt-transport-https debian-archive-keyring + +echo "OK: base packages installed" diff --git a/infra/phases/02-users.sh b/infra/phases/02-users.sh new file mode 100755 index 0000000..fd90c52 --- /dev/null +++ b/infra/phases/02-users.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +# deploy user runs lyrics-api and owns its data dirs +if ! id deploy > /dev/null 2>&1; then + useradd --system --create-home --shell /bin/bash deploy +fi + +# Working dirs +install -d -o deploy -g deploy -m 755 /opt/lyrics-api +install -d -o deploy -g deploy -m 755 /opt/lyrics-api-previews +install -d -o deploy -g deploy -m 755 /var/lib/lyrics-api +install -d -o deploy -g deploy -m 755 /var/lib/lyrics-api/data +install -d -o deploy -g deploy -m 755 /var/lib/lyrics-api/backups + +# Preview infra +install -d -o root -g root -m 755 /etc/lyrics-api-previews +install -d -o root -g root -m 755 /etc/caddy/previews + +echo "OK: deploy user + working dirs" diff --git a/infra/phases/03-caddy.sh b/infra/phases/03-caddy.sh new file mode 100755 index 0000000..b6c6711 --- /dev/null +++ b/infra/phases/03-caddy.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -euo pipefail + +# Install Caddy from official repo (idempotent) +if ! command -v caddy > /dev/null; then + curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' \ + | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg + curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' \ + | tee /etc/apt/sources.list.d/caddy-stable.list + apt-get update + apt-get install -y caddy +fi + +# Add cloudflare DNS plugin (idempotent: caddy add-package detects duplicates) +if ! caddy list-modules 2>/dev/null | grep -q 'dns.providers.cloudflare'; then + caddy add-package github.com/caddy-dns/cloudflare +fi + +# Caddyfile (substitute placeholders at install time) +sed -e "s|__ACME_EMAIL__|${ACME_EMAIL}|g" \ + -e "s|__PRIMARY_DOMAIN__|${PRIMARY_DOMAIN}|g" \ + -e "s|__STAGING_DOMAIN__|${STAGING_DOMAIN}|g" \ + -e "s|__LOGS_DOMAIN__|${LOGS_DOMAIN}|g" \ + -e "s|__METRICS_DOMAIN__|${METRICS_DOMAIN}|g" \ + -e "s|__PREVIEW_WILDCARD__|${PREVIEW_WILDCARD}|g" \ + "$INFRA_DIR/files/caddy/Caddyfile" > /etc/caddy/Caddyfile +chmod 644 /etc/caddy/Caddyfile +chown root:root /etc/caddy/Caddyfile + +# Drop-in to source CF_API_TOKEN from /etc/caddy.env (mode 600) +install -d -m 755 /etc/systemd/system/caddy.service.d +install -m 644 -o root -g root \ + "$INFRA_DIR/files/caddy/caddy.service.d/override.conf" \ + /etc/systemd/system/caddy.service.d/override.conf + +# /etc/caddy.env: mode 600, root:caddy. Caddy must exist as a user by now (apt creates it). +install -m 600 -o root -g caddy /dev/null /etc/caddy.env +printf 'CF_API_TOKEN=%s\n' "$CF_API_TOKEN" > /etc/caddy.env +chmod 600 /etc/caddy.env +chown root:caddy /etc/caddy.env + +systemctl daemon-reload +systemctl enable --now caddy +systemctl reload caddy + +echo "OK: caddy configured (CF_API_TOKEN in /etc/caddy.env, mode 600 root:caddy)" diff --git a/infra/phases/04-lyrics-api.sh b/infra/phases/04-lyrics-api.sh new file mode 100755 index 0000000..16c5ada --- /dev/null +++ b/infra/phases/04-lyrics-api.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -euo pipefail + +# Install systemd units. The binary itself is deployed separately +# (out of band: scp from CI, or rebuild from source on the box). +# bootstrap places a placeholder if no binary exists, so systemctl can validate the unit. + +install -m 644 -o root -g root \ + "$INFRA_DIR/files/lyrics-api/lyrics-api.service" \ + /etc/systemd/system/lyrics-api.service + +install -d -m 755 /etc/systemd/system/lyrics-api.service.d +install -m 644 -o root -g root \ + "$INFRA_DIR/files/lyrics-api/memory.conf" \ + /etc/systemd/system/lyrics-api.service.d/memory.conf + +# Preview template unit (used by .github/workflows/preview.yml via preview-deploy.sh) +install -m 644 -o root -g root \ + "$INFRA_DIR/../scripts/preview/lyrics-api@.service" \ + /etc/systemd/system/lyrics-api@.service + +systemctl daemon-reload + +# Don't start lyrics-api here: the binary may not exist yet on a fresh box. +# After this phase, deploy the binary to /opt/lyrics-api/lyrics-api-go and +# Infisical (phase 05) will write /etc/lyrics-api.env, which triggers the first start. + +if [ -x /opt/lyrics-api/lyrics-api-go ] && [ -f /etc/lyrics-api.env ]; then + systemctl enable --now lyrics-api + echo "OK: lyrics-api started (binary + env present)" +else + systemctl enable lyrics-api || true + echo "OK: lyrics-api unit installed (deferred start - binary or /etc/lyrics-api.env not present yet)" +fi diff --git a/infra/phases/05-infisical.sh b/infra/phases/05-infisical.sh new file mode 100755 index 0000000..e2184d7 --- /dev/null +++ b/infra/phases/05-infisical.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -euo pipefail + +# Install Infisical CLI from official apt repo +if ! command -v infisical > /dev/null; then + curl -1sLf 'https://artifacts-cli.infisical.com/setup.deb.sh' | bash + apt-get install -y infisical +fi + +install -d -m 755 /etc/infisical-agent + +# Auth files (mode 600, root only) +printf '%s' "$INFISICAL_CLIENT_ID" > /etc/infisical-agent/client-id +printf '%s' "$INFISICAL_CLIENT_SECRET" > /etc/infisical-agent/client-secret +chmod 600 /etc/infisical-agent/client-id /etc/infisical-agent/client-secret +chown root:root /etc/infisical-agent/client-id /etc/infisical-agent/client-secret + +# Agent config +install -m 644 -o root -g root "$INFRA_DIR/files/infisical/agent.yaml" /etc/infisical-agent/agent.yaml + +# Template - substitute project id + env from secrets.env +sed -e "s|__INFISICAL_PROJECT_ID__|${INFISICAL_PROJECT_ID}|g" \ + -e "s|__INFISICAL_ENV__|${INFISICAL_ENV}|g" \ + "$INFRA_DIR/files/infisical/lyrics-api.env.tpl" > /etc/infisical-agent/lyrics-api.env.tpl +chmod 644 /etc/infisical-agent/lyrics-api.env.tpl + +# Reload script + systemd unit +install -m 755 -o root -g root \ + "$INFRA_DIR/files/infisical/lyrics-api-reload" \ + /usr/local/bin/lyrics-api-reload + +install -m 644 -o root -g root \ + "$INFRA_DIR/files/infisical/infisical-agent.service" \ + /etc/systemd/system/infisical-agent.service + +systemctl daemon-reload +systemctl enable --now infisical-agent + +# Wait briefly for first sync, then trigger reload script if staging file exists +sleep 5 +if [ -s /etc/lyrics-api.env.staging ]; then + /usr/local/bin/lyrics-api-reload || echo "WARN: initial reload failed, check journalctl -t infisical-agent" +fi + +echo "OK: infisical-agent running and synced" diff --git a/infra/phases/06-beszel.sh b/infra/phases/06-beszel.sh new file mode 100755 index 0000000..df24527 --- /dev/null +++ b/infra/phases/06-beszel.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -euo pipefail + +# Install Beszel agent via official installer (creates beszel user + systemd unit) +if ! systemctl list-unit-files beszel-agent.service > /dev/null 2>&1; then + curl -sL https://get.beszel.dev/agent -o /tmp/install-agent.sh + chmod +x /tmp/install-agent.sh + /tmp/install-agent.sh \ + -p "$BESZEL_AGENT_PORT" \ + -k "$BESZEL_AGENT_KEY" \ + -t "$BESZEL_AGENT_TOKEN" \ + --hub-url "$BESZEL_HUB_URL" + rm -f /tmp/install-agent.sh +fi + +# Reconcile env vars in the unit (in case secrets rotated) +UNIT=/etc/systemd/system/beszel-agent.service +if [ -f "$UNIT" ]; then + sed -i \ + -e "s|^Environment=\"PORT=.*\"|Environment=\"PORT=${BESZEL_AGENT_PORT}\"|" \ + -e "s|^Environment=\"KEY=.*\"|Environment=\"KEY=${BESZEL_AGENT_KEY}\"|" \ + -e "s|^Environment=\"TOKEN=.*\"|Environment=\"TOKEN=${BESZEL_AGENT_TOKEN}\"|" \ + -e "s|^Environment=\"HUB_URL=.*\"|Environment=\"HUB_URL=${BESZEL_HUB_URL}\"|" \ + "$UNIT" + systemctl daemon-reload + systemctl restart beszel-agent +fi + +systemctl enable beszel-agent +echo "OK: beszel-agent registered with $BESZEL_HUB_URL" diff --git a/infra/phases/07-logdy.sh b/infra/phases/07-logdy.sh new file mode 100755 index 0000000..645627b --- /dev/null +++ b/infra/phases/07-logdy.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -euo pipefail + +# Install pinned Logdy binary +INSTALLED_VERSION="" +if [ -x /usr/local/bin/logdy ]; then + INSTALLED_VERSION=$(/usr/local/bin/logdy --version 2>/dev/null | awk '{print $NF}') +fi + +if [ "$INSTALLED_VERSION" != "$LOGDY_VERSION" ]; then + ARCH=$(dpkg --print-architecture) # arm64 or amd64 + case "$ARCH" in + arm64) LOGDY_ARCH=arm64 ;; + amd64) LOGDY_ARCH=amd64 ;; + *) echo "ERROR: unsupported arch $ARCH"; exit 1 ;; + esac + URL="https://github.com/logdyhq/logdy-core/releases/download/v${LOGDY_VERSION}/logdy_linux_${LOGDY_ARCH}" + curl -fsSL "$URL" -o /usr/local/bin/logdy + chmod 755 /usr/local/bin/logdy +fi + +# /etc/logdy.env (mode 640, deploy can read) +install -m 640 -o root -g deploy /dev/null /etc/logdy.env +printf 'LOGDY_UI_PASS=%s\n' "$LOGDY_UI_PASS" > /etc/logdy.env +chmod 640 /etc/logdy.env +chown root:deploy /etc/logdy.env + +install -m 644 -o root -g root \ + "$INFRA_DIR/files/logdy/logdy.service" \ + /etc/systemd/system/logdy.service + +systemctl daemon-reload +systemctl enable --now logdy + +echo "OK: logdy v${LOGDY_VERSION} running on localhost:8888" diff --git a/infra/phases/08-backups.sh b/infra/phases/08-backups.sh new file mode 100755 index 0000000..e387669 --- /dev/null +++ b/infra/phases/08-backups.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +# Backup scripts +install -m 755 -o root -g root \ + "$INFRA_DIR/files/backups/backup-cache.sh" \ + /usr/local/bin/backup-cache.sh +install -m 755 -o root -g root \ + "$INFRA_DIR/files/backups/upload-backup.sh" \ + /usr/local/bin/upload-backup.sh + +# rclone config for deploy user (mode 600) +install -d -o deploy -g deploy -m 700 /home/deploy/.config +install -d -o deploy -g deploy -m 700 /home/deploy/.config/rclone +sed -e "s|__B2_KEY_ID__|${B2_KEY_ID}|g" \ + -e "s|__B2_APP_KEY__|${B2_APP_KEY}|g" \ + "$INFRA_DIR/files/rclone/rclone.conf.example" > /home/deploy/.config/rclone/rclone.conf +chmod 600 /home/deploy/.config/rclone/rclone.conf +chown deploy:deploy /home/deploy/.config/rclone/rclone.conf + +# Crontab: install fragment for deploy user (idempotent: replaces wholesale) +crontab -u deploy "$INFRA_DIR/files/backups/crontab.fragment" + +echo "OK: backup scripts + rclone B2 config + deploy crontab" diff --git a/infra/phases/09-firewall.sh b/infra/phases/09-firewall.sh new file mode 100755 index 0000000..a552807 --- /dev/null +++ b/infra/phases/09-firewall.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -euo pipefail + +# UFW: allow ssh + http + https only +ufw --force reset +ufw default deny incoming +ufw default allow outgoing +ufw allow 22/tcp +ufw allow 80/tcp +ufw allow 443/tcp +ufw --force enable + +# fail2ban: defaults are fine (sshd jail enabled out of the box) +systemctl enable --now fail2ban + +echo "OK: ufw active (22/80/443), fail2ban active" diff --git a/infra/phases/10-preview-deploy.sh b/infra/phases/10-preview-deploy.sh new file mode 100755 index 0000000..1797c69 --- /dev/null +++ b/infra/phases/10-preview-deploy.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -euo pipefail + +# Preview deploy/teardown scripts (referenced by .github/workflows/preview.yml) +install -m 755 -o root -g root \ + "$INFRA_DIR/../scripts/preview/preview-deploy.sh" \ + /usr/local/bin/preview-deploy.sh +install -m 755 -o root -g root \ + "$INFRA_DIR/../scripts/preview/preview-teardown.sh" \ + /usr/local/bin/preview-teardown.sh + +# Config consumed by preview-deploy.sh: derive the wildcard base from secrets.env's PREVIEW_WILDCARD (strip leading "*.") +PREVIEW_BASE="${PREVIEW_WILDCARD#\*.}" +install -m 644 -o root -g root /dev/null /etc/preview-deploy.env +printf 'PREVIEW_BASE=%s\n' "$PREVIEW_BASE" > /etc/preview-deploy.env + +# Sudoers entry: deploy user can run preview scripts NOPASSWD +install -m 440 -o root -g root \ + "$INFRA_DIR/files/lyrics-api/sudoers.deploy-preview" \ + /etc/sudoers.d/deploy-preview + +# Validate sudoers syntax +visudo -c -f /etc/sudoers.d/deploy-preview > /dev/null + +echo "OK: preview deploy/teardown scripts + sudoers entry + preview-deploy.env" diff --git a/infra/secrets.env.example b/infra/secrets.env.example new file mode 100644 index 0000000..5ca4501 --- /dev/null +++ b/infra/secrets.env.example @@ -0,0 +1,38 @@ +# Better Lyrics API - secrets for bootstrap.sh +# Copy this file to secrets.env on the target box and fill in every value before running bootstrap. +# secrets.env is gitignored. Source values from Bitwarden. + +# === Cloudflare (DNS-01 ACME for Caddy) === +CF_API_TOKEN= +ACME_EMAIL= + +# === Infisical (machine identity for agent that syncs /etc/lyrics-api.env) === +INFISICAL_CLIENT_ID= +INFISICAL_CLIENT_SECRET= +INFISICAL_PROJECT_ID= +INFISICAL_ENV=prod + +# === Backblaze B2 (rclone remote for off-site backups) === +B2_KEY_ID= +B2_APP_KEY= +B2_BUCKET=lyrics-api-backups + +# === Beszel (agent registers with hub) === +BESZEL_HUB_URL= +BESZEL_AGENT_KEY= +BESZEL_AGENT_TOKEN= +BESZEL_AGENT_PORT=45876 + +# === Logdy (basic auth for log viewer UI) === +LOGDY_UI_PASS= + +# === Server identity (used by Caddy / docs / verification) === +PUBLIC_IP= +PRIMARY_DOMAIN= +STAGING_DOMAIN= +LOGS_DOMAIN= +METRICS_DOMAIN= +PREVIEW_WILDCARD= + +# === Pinned versions (bump deliberately, not silently) === +LOGDY_VERSION=0.17.0 diff --git a/scripts/preview/preview-deploy.sh b/scripts/preview/preview-deploy.sh index dbbea7a..7546a2b 100755 --- a/scripts/preview/preview-deploy.sh +++ b/scripts/preview/preview-deploy.sh @@ -4,6 +4,11 @@ set -euo pipefail PR="${1:-}" [[ "$PR" =~ ^[0-9]+$ ]] || { echo "Usage: $0 "; exit 2; } +# PREVIEW_BASE is the wildcard zone, e.g. "preview.api.example.com" so PR 42 lands at "pr-42.preview.api.example.com". +# Written by infra/phases/10-preview-deploy.sh from $PREVIEW_WILDCARD in secrets.env. +[ -f /etc/preview-deploy.env ] && source /etc/preview-deploy.env +: "${PREVIEW_BASE:?PREVIEW_BASE not set; expected /etc/preview-deploy.env to define it}" + UNIT="lyrics-api@${PR}.service" PORT=$((9000 + PR)) DIR="/opt/lyrics-api-previews/pr-${PR}" @@ -39,7 +44,7 @@ chmod 640 "$ENV_FILE" chown root:deploy "$ENV_FILE" cat > "$CADDY_FILE" <