deploy.yml

name: Deploy

on:
  workflow_dispatch:
    inputs:
      target:
        description: Deployment target
        type: choice
        required: true
        default: home
        options: [home, vps, both]

permissions:
  contents: read

concurrency:
  group: honeypot-production
  cancel-in-progress: false

jobs:
  home:
    if: inputs.target == 'home' || inputs.target == 'both'
    runs-on: [self-hosted, linux, x64, honeypot-home]
    environment: production-home
    steps:
      - uses: actions/checkout@v7
      # #258, updated by #1502: this rsync's only remaining direct consumer
      # in THIS workflow is the ghidra-worker.py host-level re-sync step
      # right below it (bash /opt/stacks/apiary/analysis/ghidra/
      # install-analysis-host.sh --host-files-only) -- every stack that
      # used to build from or bind-mount an absolute path under
      # /opt/stacks/apiary is now either an Arcane-managed directory-aware
      # Git sync with its own self-contained build context (32 stacks,
      # arcane/home/<name>/) or one of the 6 stacks that were already
      # self-contained and are provisioned by scripts/install-homeserver.sh
      # instead of this file. Kept rather than removed: still the simplest
      # way to give the ghidra-worker.py step a fresh checkout, and
      # `docker compose -f compose.yml config --quiet` below is a cheap
      # sanity check that the (deliberately empty, see its own header)
      # root docker-compose.yml is still valid YAML/Compose syntax.
      #
      # Historical note this comment used to carry: an earlier version of
      # this step ran near the end of the job and every stack above it
      # silently built from the *previous* deploy's checkout, one deploy
      # stale (#1103). Not a live concern for what this step does now, but
      # worth remembering if something gets added above the rsync again --
      # anything that reads from $destination has to run after it.
      - name: Synchronize APIARY source without runtime state
        shell: bash
        run: |
          destination=/opt/stacks/apiary
          # analysis/ghidra/revdeck/ and sandbox/windows/packer/pxe/ipxe*
          # are hand-vendored/build-generated on the host, not tracked in
          # this repo (see analysis/ghidra/revdeck-proxyfix/'s own doc
          # comment and prepare-pxe.sh respectively) -- without excluding
          # them, --delete-delay either fails outright (a populated
          # directory rsync can't recursively delete under this call) or,
          # worse, silently deletes a hand-built ipxe.efi/webui checkout on
          # every future deploy. Found live: this rsync step failed here
          # once these existed on disk, exit 23.
          #
          # analysis/geoip/ itself (not just its *.mmdb files) is excluded
          # wholesale, not just the mmdb pattern: nothing under this path is
          # git-tracked, and geoipupdate (root-owned container, #1258) also
          # writes a root:root .geoipupdate.lock into the root:root
          # directory it owns -- the deploy-runner user can't unlink a file
          # inside a directory it has no write permission on, so a narrower
          # exclude that missed this file broke every deploy once the
          # geoip-update profile started actually running (#1226 fix).
          rsync -a --delete-delay \
            --exclude '.git/' --exclude '.github/' --exclude '.env' \
            --exclude 'logs/' --exclude 'state/' --exclude 'dashboard-state/' \
            --exclude 'analysis/geoip/' --exclude 'sandbox/results/' \
            --exclude 'analysis/ghidra/revdeck/' \
            --exclude 'sandbox/windows/packer/pxe/ipxe/' \
            --exclude 'sandbox/windows/packer/pxe/ipxe.efi' \
            --exclude 'sandbox/windows/packer/pxe/ipxe.efi.unsigned' \
            ./ "$destination/"
          cp "$destination/docker-compose.yml" "$destination/compose.yml"
          cd "$destination"
          test ! -f .env || chmod 600 .env
          docker compose -f compose.yml config --quiet

      # #1406: ghidra-worker.py and its siblings are a host-level systemd
      # service (analysis/ghidra/install-analysis-host.sh), not a Docker
      # Compose service the sync loops below touch -- the rsync step above
      # only updates the /opt/stacks/apiary checkout those files are
      # installed FROM, it never re-runs the installer that actually copies
      # them to /opt/honeypot-ghidra and restarts the long-running unit.
      # Confirmed live (#1406's own issue): a merged worker fix sat on
      # main, deploy.yml reported success, and the running copy stayed
      # stale for days until manually re-synced.
      #
      # This runner has no passwordless sudo, and the installer needs real
      # root against the real host systemd -- nsenter into PID 1's
      # namespaces from a throwaway privileged container is this repo's
      # established escape hatch for exactly that (same trick already used
      # for host-level directory permission fixes this session).
      #
      # Conditional on the unit already existing: this step re-SYNCs an
      # already-installed worker, it must not be the thing that newly
      # provisions one on a host where ENABLE_GPU_STACK=false deliberately
      # skipped it during install-homeserver.sh.
      - name: Re-sync ghidra-worker.py and siblings (host-level systemd, not Compose)
        shell: bash
        run: |
          if docker run --rm --privileged --pid=host debian:bookworm-slim \
               nsenter --target 1 --mount --uts --ipc --net --pid -- \
               systemctl list-unit-files honeypot-ghidra-worker.path \
               >/dev/null 2>&1; then
            docker run --rm --privileged --pid=host debian:bookworm-slim \
              nsenter --target 1 --mount --uts --ipc --net --pid -- \
              bash /opt/stacks/apiary/analysis/ghidra/install-analysis-host.sh --host-files-only
          else
            echo "honeypot-ghidra-worker.path not installed on this host -- skipping re-sync (ENABLE_GPU_STACK=false at provisioning time, or not yet provisioned)"
          fi

      # #1185: replaces Dockge as the stack-management UI. Same
      # fail-closed-until-provisioned shape as honeypot-keycloak above --
      # ENCRYPTION_KEY/JWT_SECRET/OIDC_CLIENT_SECRET have no *_FILE variant
      # Arcane supports, so they live in this stack's own .env (never
      # synced or overwritten here, same as every other stack's .env).
      # Deployed after honeypot-keycloak, not before: OIDC_ISSUER_URL in
      # that .env points at the same realm this step's own OIDC client
      # depends on already existing.
      - name: Synchronize honeypot-arcane
        shell: bash
        run: |
          destination=/var/dockge/stacks/honeypot-arcane
          install -d -m 755 "$destination"
          cp docker-compose.arcane.yml "$destination/compose.yml"
          cd "$destination"
          if test -f .env; then
            chmod 600 .env
            docker compose -f compose.yml config --quiet
            docker compose -f compose.yml up -d
          else
            echo "honeypot-arcane synchronized but not started: provision .env (ARCANE_ENCRYPTION_KEY, ARCANE_JWT_SECRET, OIDC_ISSUER_URL, ARCANE_OIDC_CLIENT_SECRET) first"
          fi

      # #1502: honeypot-init/keycloak/conpot and the 20-stack loop that used
      # to run here (plus ip-enrichment-worker/agent-intrusion-worker/
      # payload-analysis/utilities/elk/dashboard below them) are gone --
      # those 29 stacks (30 counting honeypot-arcane's own bootstrap
      # deploy above, which stays) are now Arcane-managed directory-aware
      # Git syncs (arcane/manifests/home-production.json), not something
      # this workflow copies files for. Running this cp/build loop
      # alongside Arcane's own sync would be exactly the "second writer"
      # problem #1502's own migration review flagged: two independent
      # mechanisms racing to own the same on-host directory. What's left
      # for this job to do now is a lightweight CI gate: every stack listed
      # in the manifest -- the 32 that migrated under arcane/home/ plus the
      # 6 that were already self-contained (auth-events-worker, llm-worker,
      # ml-worker, ghidra, ghosts, pihole) and stayed at their existing
      # path -- still has a syntactically valid compose file, so a broken
      # commit on this branch fails here instead of silently sitting
      # unsynced (or, worse, half-synced) on the Arcane side. None of those
      # 38 are deployed by this workflow at all any more; the 6
      # self-contained ones are still provisioned by
      # scripts/install-homeserver.sh's own dedicated steps on a
      # from-scratch install, not by anything in this file.
      - name: Validate every Arcane-managed stack's compose file
        shell: bash
        run: |
          # A stack with a genuinely required secret (canarytokens, ghosts,
          # pihole, keycloak, arcane, ...) makes `docker compose config`
          # fail on purpose when nothing provides a real value -- that's
          # the fail-closed behavior #1502's own migration deliberately
          # kept, not something CI should route around with a hand-
          # maintained list of every required var across 38 stacks (which
          # would silently drift the moment a new one is added). Instead:
          # retry with whatever variable Compose's own error names as
          # missing set to a throwaway placeholder, until config succeeds
          # or nothing is left to fill in -- validates real structure/
          # syntax without needing to know any stack's real secrets.
          validate_one() {
            local path="$1"
            local -a extra_env=()
            local attempt=0 err=""
            while (( attempt < 10 )); do
              err=$(env "${extra_env[@]}" docker compose -f "$path" config --quiet 2>&1)
              [ -z "$err" ] && return 0
              local missing
              missing=$(echo "$err" | grep -oE 'required variable [A-Za-z_][A-Za-z0-9_]* is missing' | head -1 | awk '{print $3}')
              [ -z "$missing" ] && { echo "$err" >&2; return 1; }
              extra_env+=("$missing=ci-placeholder-value")
              attempt=$((attempt + 1))
            done
            echo "$err" >&2
            return 1
          }

          fail=0
          while IFS=$'\t' read -r name path; do
            if [ ! -f "$path" ]; then
              echo "::error::$name: $path listed in the manifest does not exist"
              fail=1
              continue
            fi
            if ! validate_one "$path"; then
              echo "::error::$name: docker compose config failed for $path (see above)"
              fail=1
            fi
          done < <(jq -r '.[] | [.syncName, .dockerComposePath] | @tsv' arcane/manifests/home-production.json)
          exit "$fail"

  vps:
    if: inputs.target == 'vps' || inputs.target == 'both'
    runs-on: ubuntu-latest
    environment: production-vps
    steps:
      - uses: actions/checkout@v7
      - name: Configure deployment key
        shell: bash
        env:
          VPS_SSH_KEY: ${{ secrets.VPS_SSH_KEY }}
        run: |
          install -d -m 700 ~/.ssh
          printf '%s\n' "$VPS_SSH_KEY" > ~/.ssh/honeypot_vps
          chmod 600 ~/.ssh/honeypot_vps
      - name: Synchronize and restart plain Compose VPS stack
        shell: bash
        env:
          VPS_HOST: ${{ secrets.VPS_HOST }}
          VPS_USER: ${{ secrets.VPS_USER }}
          VPS_PORT: ${{ secrets.VPS_PORT }}
        run: |
          ssh_options=(-i ~/.ssh/honeypot_vps -p "${VPS_PORT:-2222}" -o StrictHostKeyChecking=accept-new)

          # Snapshot the environment-specific files before touching the host, so
          # a mistake here is recoverable without reissuing certificates.
          ssh "${ssh_options[@]}" "${VPS_USER:-root}@${VPS_HOST}" \
            'set -eu; cd /root/vps 2>/dev/null || exit 0
             stamp=$(date -u +%Y%m%dT%H%M%SZ)
             install -d -m 700 /root/vps-backups
             set -- ; for p in traefik/certs traefik/dynamic.yml .env secrets; do
               test -e "$p" && set -- "$@" "$p"
             done
             test "$#" -gt 0 && tar czf "/root/vps-backups/pre-deploy-$stamp.tar.gz" "$@"
             ls -1t /root/vps-backups/pre-deploy-*.tar.gz | tail -n +11 | xargs -r rm -f'

          # --delete-delay removes anything on the VPS that is absent from the
          # repository. Things that must therefore never be synchronized by
          # the bulk rsync below:
          #   .env                  secrets, host-specific
          #   traefik/certs/        issued certificates — deleting these takes
          #                         TLS down and they cannot be regenerated here
          #   traefik/dynamic.yml   carries the real domain; the committed copy
          #                         is a honeypot.example placeholder -- see
          #                         the dedicated step below, which deploys
          #                         this one deliberately differently (not
          #                         "never touch it") now that DOMAIN is a
          #                         secret instead of tribal knowledge.
          #   secrets/              per-gateway cookie-secret/client-secret
          #                         files (OIDC_SECRETS_DIR=./secrets/oidc in
          #                         vps/.env.example), git-ignored and never
          #                         present in this checkout at all --
          #                         --delete-delay treats "absent from the
          #                         source" as "delete it", which took down
          #                         all seven oauth2-proxy gateways
          #                         (Kibana/EveBox/Arkime/TANNER/RevDeck/
          #                         Dockge/Traefik) in a real incident this
          #                         exclude list didn't cover at the time.
          #                         client-secret must be recovered from
          #                         Keycloak's own stored value (kcadm get
          #                         clients/<id>/client-secret), never
          #                         regenerated -- it has to match what's
          #                         already registered per client.
          rsync -az --delete-delay \
            --exclude '.env' \
            --exclude 'traefik/certs/' \
            --exclude 'traefik/dynamic.yml' \
            --exclude 'secrets/' \
            -e "ssh ${ssh_options[*]}" vps/ "${VPS_USER:-root}@${VPS_HOST}:/root/vps/"

          ssh "${ssh_options[@]}" "${VPS_USER:-root}@${VPS_HOST}" \
            'cd /root/vps && { test ! -f .env || chmod 600 .env; } && docker compose -f docker-compose.yml config --quiet && docker compose -f docker-compose.yml up -d --build'

      - name: Deploy Traefik dynamic config with the real domain substituted
        shell: bash
        env:
          VPS_HOST: ${{ secrets.VPS_HOST }}
          VPS_USER: ${{ secrets.VPS_USER }}
          VPS_PORT: ${{ secrets.VPS_PORT }}
          DOMAIN: ${{ secrets.DOMAIN }}
        run: |
          # traefik/dynamic.yml is excluded from the bulk rsync above and
          # deployed here instead, deliberately differently. Traefik's file
          # provider has no env-var substitution the way docker-compose's
          # ${VAR} syntax already gives every other host-specific value in
          # this repo (vps/docker-compose.yml's OIDC_ISSUER_URL/OIDC_PUBLIC_DOMAIN
          # etc, which needed no change here) -- so the committed file is a
          # placeholder (*.honeypot.example) and the DOMAIN secret
          # substitutes it here, in CI, instead of via a manual live-only
          # edit no one wrote down.
          #
          # The substituted file is written to the live path with a plain
          # in-place `cat`, NOT copied via the rsync above (or any rename-
          # into-place, including `mv`). Traefik's compose service bind-
          # mounts this file at a single path
          # (traefik/dynamic.yml:/etc/traefik/dynamic.yml:ro), which Docker
          # binds to the specific INODE present when the container started,
          # not the path -- a rename-based replacement repoints the host
          # path at a new inode while the already-running container keeps
          # reading the old one, with no error of any kind to signal the
          # mismatch. Learned this live: after "fixing" a broken domain this
          # way once, the site kept serving the wrong config to real
          # visitors even though every host-side check (cat, diff) showed
          # the new file as correct. `cat new > existing_path` truncates and
          # rewrites the EXISTING inode in place, which the bind mount does
          # see immediately -- Traefik hot-reloads it with no restart
          # needed (dynamic.yml's own header comment: "changes apply
          # instantly without restarting Traefik").
          if [ -z "$DOMAIN" ]; then
            echo "::error::DOMAIN secret is not set -- refusing to deploy a config with placeholder domains"
            exit 1
          fi
          sed "s/honeypot\.example/${DOMAIN}/g" vps/traefik/dynamic.yml > /tmp/dynamic.yml.deployable
          python3 -c "import yaml; yaml.safe_load(open('/tmp/dynamic.yml.deployable'))"
          if grep -q "honeypot.example" /tmp/dynamic.yml.deployable; then
            echo "::error::generated dynamic.yml still contains placeholder domains after substitution"
            exit 1
          fi
          scp -i ~/.ssh/honeypot_vps -P "${VPS_PORT:-2222}" -o StrictHostKeyChecking=accept-new \
            /tmp/dynamic.yml.deployable "${VPS_USER:-root}@${VPS_HOST}:/tmp/dynamic.yml.deployable"
          # ssh_options is a bash array from the "Synchronize and restart"
          # step above -- array variables don't survive across separate
          # GitHub Actions steps (each `run:` is its own shell process), so
          # referencing it here without redefining it silently expanded to
          # nothing: no -i, no -p, no -o StrictHostKeyChecking=accept-new.
          # That dropped the deploy key and the port entirely and fell back
          # to ssh's default (interactive/no-tty) host-key policy, which
          # fails outright as "Host key verification failed" against a host
          # this job's own ephemeral known_hosts has never seen -- exactly
          # what happened live (2026-08-15) the first time this step ran
          # after the deploy key itself was fixed. Every other step that
          # uses "${ssh_options[@]}" either defines it in the same step
          # (this one now included) or reuses it later within the *same*
          # step it was defined in (the scp above it, and the "Synchronize"
          # step's own later ssh call) -- this was the one place that
          # crossed a step boundary.
          ssh_options=(-i ~/.ssh/honeypot_vps -p "${VPS_PORT:-2222}" -o StrictHostKeyChecking=accept-new)
          ssh "${ssh_options[@]}" "${VPS_USER:-root}@${VPS_HOST}" \
            'cat /tmp/dynamic.yml.deployable > /root/vps/traefik/dynamic.yml && rm -f /tmp/dynamic.yml.deployable'

      - name: Verify the live Traefik configuration survived
        shell: bash
        env:
          VPS_HOST: ${{ secrets.VPS_HOST }}
          VPS_USER: ${{ secrets.VPS_USER }}
          VPS_PORT: ${{ secrets.VPS_PORT }}
        run: |
          ssh_options=(-i ~/.ssh/honeypot_vps -p "${VPS_PORT:-2222}" -o StrictHostKeyChecking=accept-new)
          ssh "${ssh_options[@]}" "${VPS_USER:-root}@${VPS_HOST}" 'set -eu
            cd /root/vps
            fail=0
            for f in traefik/certs/origin.pem traefik/certs/origin-key.pem traefik/dynamic.yml; do
              if [ ! -s "$f" ]; then echo "::error::$f is missing or empty after deploy"; fail=1; fi
            done
            if grep -q "honeypot.example" traefik/dynamic.yml 2>/dev/null; then
              echo "::error::traefik/dynamic.yml contains placeholder domains — routing is down"
              fail=1
            fi
            if [ -s traefik/certs/origin.pem ] && ! openssl x509 -in traefik/certs/origin.pem -noout >/dev/null 2>&1; then
              echo "::error::traefik/certs/origin.pem is not a parseable certificate"
              fail=1
            fi
            exit "$fail"'