From 2dec7e7096fd3aa46f395538c3db8f6c728515c4 Mon Sep 17 00:00:00 2001 From: Michal Landsman Date: Mon, 3 Aug 2026 18:04:43 +0200 Subject: [PATCH 1/2] devops: run the browser checks on every pull request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e suite was local-only, so the one thing that proves the app renders and behaves — as opposed to generating the right HTML — never ran on a change unless someone remembered to run it. `check` boots the app but stops at the login page. Linux only and on its own job: the checks drive frankenphp and a headless chromium, not the native shell, so a second platform would re-run the same assertions at 10x, and a red e2e should read as a broken app rather than as a broken lint. --- .github/workflows/build.yml | 56 +++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 37fcaf4..92d5e3b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,6 +73,62 @@ jobs: # installed and no credentials. - run: make security + # The browser checks, on their own job for the same reason `quality` is on its own: a red + # e2e should read as a broken app, not as a broken lint. Linux only — the checks drive + # frankenphp and a headless chromium, neither of which is the native shell, so a second + # platform would re-run the same assertions at 10x the price. + e2e: + if: github.event_name == 'pull_request' && github.event.pull_request.draft != true + runs-on: ubuntu-24.04 + # docker pull + 15 checks, each with its own server and browser. The default is 6 hours, + # which on a billed private repo is what a wedged run costs before anyone notices. + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + + # The same toolchain a developer gets: node for the browser Playwright drives, and gh, + # which `mise run install` checks is logged in before it spends anything. + - uses: jdx/mise-action@v4 + + - name: Cache pinned downloads + uses: actions/cache@v6 + with: + path: | + bin + .cache + key: pinned-frankenphp-linux-x86_64-${{ hashFiles('Makefile') }} + + # Before `mise run install`: composer runs on the bundled frankenphp, so app/vendor + # cannot be built until the binary is here. + - run: make fetch FRANKEN_ASSET=frankenphp-linux-x86_64 + + # composer deps, npm deps and the chromium Playwright launches. + - run: mise run install + env: + GH_TOKEN: ${{ github.token }} + + # Playwright ships its own chromium but not the system libraries it links against, and + # the runner image only has the ones its own Chrome needs. A no-op when they all happen + # to be there — which is why it is not worth guessing. Not in `mise run install`: that + # runs on developer machines, and it installs with sudo apt. + - name: Playwright system libraries + working-directory: app/vendor/playwright-php/playwright/bin + run: npx --no-install playwright install-deps chromium + + # Boots its own postgres in docker (tests/e2e/fixture.php), so nothing to service here. + - run: make e2e FRANKEN_ASSET=frankenphp-linux-x86_64 + + # A failed assertion says which check and which selector; the screenshot says what the + # page actually looked like, and there is no other way to see that from a runner. + - if: failure() + uses: actions/upload-artifact@v7 + with: + name: e2e-screenshots + path: tests/e2e/screenshots + if-no-files-found: ignore + # Manual only. Three platforms per commit is minutes spent to learn that a one-line PHP # change is still fine, and macOS runners bill at 10x. Run it from the Actions tab: tick # the build box for artifacts, or name an existing tag to build that and attach the From bcd6afc4d4f6ad66aab6f6d97e186f87b9395061 Mon Sep 17 00:00:00 2001 From: Michal Landsman Date: Mon, 3 Aug 2026 18:08:28 +0200 Subject: [PATCH 2/2] devops: move the browser checks to a nightly workflow of their own Per pull request was the wrong cadence: 15 checks, each booting a postgres, a server and a chromium, spent on every push to re-prove what the last run proved about code that mostly does not touch the UI. Once a night still catches a regression by the next morning, and the Actions tab has the button for when the answer is wanted now. Its own file rather than a job in build.yml, because nothing here shares a step with the packaging matrix and a schedule bolted onto that file would have every job in it carrying an `if:` to say which trigger it answers to. The one pull_request trigger left is `paths: [.github/workflows/e2e.yml]`: a nightly that has never run is a nightly nobody knows is broken, and the first honest run of it should not be at 21:30 with nobody watching. --- .github/workflows/build.yml | 56 --------------------------- .github/workflows/e2e.yml | 77 +++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 92d5e3b..37fcaf4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,62 +73,6 @@ jobs: # installed and no credentials. - run: make security - # The browser checks, on their own job for the same reason `quality` is on its own: a red - # e2e should read as a broken app, not as a broken lint. Linux only — the checks drive - # frankenphp and a headless chromium, neither of which is the native shell, so a second - # platform would re-run the same assertions at 10x the price. - e2e: - if: github.event_name == 'pull_request' && github.event.pull_request.draft != true - runs-on: ubuntu-24.04 - # docker pull + 15 checks, each with its own server and browser. The default is 6 hours, - # which on a billed private repo is what a wedged run costs before anyone notices. - timeout-minutes: 20 - permissions: - contents: read - steps: - - uses: actions/checkout@v7 - - # The same toolchain a developer gets: node for the browser Playwright drives, and gh, - # which `mise run install` checks is logged in before it spends anything. - - uses: jdx/mise-action@v4 - - - name: Cache pinned downloads - uses: actions/cache@v6 - with: - path: | - bin - .cache - key: pinned-frankenphp-linux-x86_64-${{ hashFiles('Makefile') }} - - # Before `mise run install`: composer runs on the bundled frankenphp, so app/vendor - # cannot be built until the binary is here. - - run: make fetch FRANKEN_ASSET=frankenphp-linux-x86_64 - - # composer deps, npm deps and the chromium Playwright launches. - - run: mise run install - env: - GH_TOKEN: ${{ github.token }} - - # Playwright ships its own chromium but not the system libraries it links against, and - # the runner image only has the ones its own Chrome needs. A no-op when they all happen - # to be there — which is why it is not worth guessing. Not in `mise run install`: that - # runs on developer machines, and it installs with sudo apt. - - name: Playwright system libraries - working-directory: app/vendor/playwright-php/playwright/bin - run: npx --no-install playwright install-deps chromium - - # Boots its own postgres in docker (tests/e2e/fixture.php), so nothing to service here. - - run: make e2e FRANKEN_ASSET=frankenphp-linux-x86_64 - - # A failed assertion says which check and which selector; the screenshot says what the - # page actually looked like, and there is no other way to see that from a runner. - - if: failure() - uses: actions/upload-artifact@v7 - with: - name: e2e-screenshots - path: tests/e2e/screenshots - if-no-files-found: ignore - # Manual only. Three platforms per commit is minutes spent to learn that a one-line PHP # change is still fine, and macOS runners bill at 10x. Run it from the Actions tab: tick # the build box for artifacts, or name an existing tag to build that and attach the diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..5a579d6 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,77 @@ +name: e2e + +# Nightly, not per pull request. The browser checks boot a postgres, a server and a chromium +# per file — minutes, on every push, to re-prove what the last run already proved about code +# that mostly does not touch the UI. Once a night catches the same regression by morning, +# and the Actions tab has the button for when you want it now. +# +# Its own workflow rather than a job in build.yml: nothing here shares a step with the +# packaging matrix, and a schedule bolted onto that file would have every job in it carrying +# an `if:` to say which trigger it answers to. +on: + schedule: + # 21:30 UTC — 23:30 in Prague through the summer, 22:30 once the clocks go back. GitHub + # cron is UTC and has no timezone, so the choice is which half of the year to be right in. + - cron: "30 21 * * *" + workflow_dispatch: + # Only when this file itself changes. A nightly that has never run is a nightly nobody + # knows is broken, and the first honest run must not be at 21:30 unwatched. + pull_request: + paths: [".github/workflows/e2e.yml"] + +jobs: + e2e: + runs-on: ubuntu-24.04 + # docker pull, then 15 checks each with its own server and browser — ~5 minutes in + # practice. The default is 6 hours, which is what a wedged run costs before anyone looks. + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + + # The same toolchain a developer gets: node for the browser Playwright drives, and gh, + # which `mise run install` checks is logged in before it spends anything. + - uses: jdx/mise-action@v4 + + # frankenphp is a ~180 MB download, pinned in the Makefile — so is everything else + # under .cache, and the key only has to change when a pin does. + - name: Cache pinned downloads + uses: actions/cache@v6 + with: + path: | + bin + .cache + key: pinned-frankenphp-linux-x86_64-${{ hashFiles('Makefile') }} + + # Before `mise run install`: composer runs on the bundled frankenphp, so app/vendor + # cannot be built until the binary is here. + - run: make fetch FRANKEN_ASSET=frankenphp-linux-x86_64 + + # composer deps, npm deps, and the chromium Playwright launches. + - run: mise run install + env: + GH_TOKEN: ${{ github.token }} + + # Playwright ships its own chromium but not the system libraries it links against, and + # the runner image only has the ones its own Chrome needs. A no-op when they all happen + # to be there, which is why it is not worth guessing. Deliberately not in + # `mise run install`: that runs on developer machines, and this installs with sudo apt. + - name: Playwright system libraries + working-directory: app/vendor/playwright-php/playwright/bin + run: npx --no-install playwright install-deps chromium + + # Boots its own postgres in docker (tests/e2e/fixture.php), so there is no service to + # declare here. Linux only, and that is the whole story: the checks drive frankenphp + # and a headless chromium, never the native shell, so a second platform would re-run + # the same assertions — at 10x on macOS. + - run: make e2e FRANKEN_ASSET=frankenphp-linux-x86_64 + + # A failed assertion names the check and the selector; the screenshot is the only way + # to see what the page actually looked like on a runner nobody was watching. + - if: failure() + uses: actions/upload-artifact@v7 + with: + name: e2e-screenshots + path: tests/e2e/screenshots + if-no-files-found: ignore