From c0064672715e5cb1560f75bb0f4c1909a2fa917f Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Fri, 10 Apr 2026 13:42:12 +0200 Subject: [PATCH 01/27] feat: centralize module testing --- .github/workflows/test-module.yml | 73 +++++++++++++++++++ .../workflows/test-on-digitalocean-infra.yml | 32 +++++++- scripts/test-module.sh | 48 ++++++++++++ scripts/tests/pythonreq.txt | 3 + 4 files changed, 152 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test-module.yml create mode 100644 scripts/test-module.sh create mode 100644 scripts/tests/pythonreq.txt diff --git a/.github/workflows/test-module.yml b/.github/workflows/test-module.yml new file mode 100644 index 0000000..112fcf6 --- /dev/null +++ b/.github/workflows/test-module.yml @@ -0,0 +1,73 @@ +name: Test module + +on: + workflow_call: + inputs: + debug_shell: + description: "Debug shell" + required: false + type: boolean + default: false + secrets: + do_token: + required: true + +jobs: + module: + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == '' }} + uses: ./.github/workflows/module-info.yml + + chooser: + runs-on: ubuntu-latest + outputs: + node_a: ${{ steps.pick.outputs.node_a }} + node_b: ${{ steps.pick.outputs.node_b }} + steps: + - id: pick + run: | + if (( $GITHUB_RUN_NUMBER % 2 )); then + echo "node_a=rl1" >> "$GITHUB_OUTPUT" + echo "node_b=dn1" >> "$GITHUB_OUTPUT" + else + echo "node_a=dn1" >> "$GITHUB_OUTPUT" + echo "node_b=rl1" >> "$GITHUB_OUTPUT" + fi + + detect_changes: + runs-on: ubuntu-latest + outputs: + run_ui_tests: ${{ steps.changes.outputs.run_ui_tests }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + fetch-depth: 2 + - id: changes + run: | + if git diff --name-only HEAD^ HEAD | grep -qE '^ui/|^build-images\.sh$'; then + echo "run_ui_tests=true" >> "$GITHUB_OUTPUT" + else + echo "run_ui_tests=false" >> "$GITHUB_OUTPUT" + fi + + run_tests: + needs: [module, chooser, detect_changes] + strategy: + fail-fast: false + matrix: + scenario: [install, update] + uses: ./.github/workflows/test-on-digitalocean-infra.yml + with: + coremodules: ${{ matrix.scenario == 'install' && format('ghcr.io/{0}/{1}:{2}', needs.module.outputs.owner, needs.module.outputs.name, needs.module.outputs.tag) || '' }} + leader_nodes: >- + ${{ + matrix.scenario == 'install' + && needs.chooser.outputs.node_a + || needs.chooser.outputs.node_b + }} + args: ${{ format('ghcr.io/{0}/{1}:{2} -v SCENARIO:{3}', needs.module.outputs.owner, needs.module.outputs.name, needs.module.outputs.tag, matrix.scenario) }} + repo_ref: ${{ needs.module.outputs.sha }} + run_ui_tests: ${{ needs.detect_changes.outputs.run_ui_tests == 'true' }} + debug_shell: ${{ inputs.debug_shell }} + secrets: + do_token: ${{ secrets.do_token }} diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index af5d0d5..cc4154d 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -5,13 +5,13 @@ on: inputs: script: required: false - description: "Name of the script to execute" + description: "Name of the script to execute. If empty, the default script from ns8-github-actions is used." type: string - default: test-module.sh + default: "" path: required: false type: string - description: "Name of the script to execute, default `test-module.sh`" + description: "Subdirectory of the module repository to use as working directory when running the test script" args: required: false type: string @@ -55,6 +55,11 @@ on: type: string default: "ci.nethserver.net" description: "DigitalOcean domain where to create the DNS records" + run_ui_tests: + required: false + type: boolean + default: false + description: "If true, run UI tests" secrets: do_token: required: true @@ -117,6 +122,15 @@ jobs: with: ref: ${{ inputs.repo_ref }} path: module + - name: "Get ns8-github-actions ref" + id: ns8_ga_ref + run: echo "ref=$(echo '${{ github.workflow_ref }}' | cut -d'@' -f2)" >> $GITHUB_OUTPUT + - name: "Checkout ns8-github-actions" + uses: actions/checkout@v6 + with: + repository: NethServer/ns8-github-actions + ref: ${{ steps.ns8_ga_ref.outputs.ref }} + path: ns8-github-actions - name: "Checkout the infrastructure repository" uses: actions/checkout@v6 with: @@ -172,14 +186,24 @@ jobs: set -e -x create-cluster ${{ matrix.node }}.leader.${{ matrix.node }}-${{ needs.info.outputs.short_sha }}.${{ env.TF_VAR_domain }}:55820 10.5.4.0/24 Nethesis,1234 EOF + - name: "Resolve test script" + id: resolve_script + run: | + if [ -n "${{ inputs.script }}" ]; then + echo "path=${{ github.workspace }}/module/${{ inputs.path }}/${{ inputs.script }}" >> "$GITHUB_OUTPUT" + else + echo "path=${{ github.workspace }}/ns8-github-actions/scripts/test-module.sh" >> "$GITHUB_OUTPUT" + fi - name: "Run tests" id: run_tests run: | - ./${{ inputs.script }} ${{ matrix.node }}.leader.${{ matrix.node }}-${{ needs.info.outputs.short_sha }}.${{ env.TF_VAR_domain }} ${{ inputs.args }} + echo "Running test script: ${{ steps.resolve_script.outputs.path }} ////" + bash "${{ steps.resolve_script.outputs.path }}" ${{ matrix.node }}.leader.${{ matrix.node }}-${{ needs.info.outputs.short_sha }}.${{ env.TF_VAR_domain }} ${{ inputs.args }} working-directory: ${{ github.workspace }}/module/${{ inputs.path }} env: SSH_KEYFILE: ${{ github.workspace }}/key COREMODULES: ${{ inputs.coremodules }} + RUN_UI_TESTS: ${{ inputs.run_ui_tests }} - name: "Save tests results" if: always() uses: actions/upload-artifact@v6 diff --git a/scripts/test-module.sh b/scripts/test-module.sh new file mode 100644 index 0000000..f901c10 --- /dev/null +++ b/scripts/test-module.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +set -e -a + +SSH_KEYFILE=${SSH_KEYFILE:-$HOME/.ssh/id_rsa} + +LEADER_NODE="${1:?missing LEADER_NODE argument}" +IMAGE_URL="${2:?missing IMAGE_URL argument}" +shift 2 + +ssh_key="$(< $SSH_KEYFILE)" +venvroot=/usr/local/venv + +echo "Test! RUN_UI_TESTS=${RUN_UI_TESTS} ////" + +podman run -i \ + --volume=.:/srv/source:z \ + --volume=rftest-cache:${venvroot}:z \ + --replace --name=rftest \ + --env=ssh_key \ + --env=venvroot \ + --env=LEADER_NODE \ + --env=IMAGE_URL \ + --env=RUN_UI_TESTS \ + docker.io/python:3.11-alpine \ + ash -l -s -- "${@}" <<'EOF' +set -e +echo "$ssh_key" > /tmp/idssh +if [ ! -x ${venvroot}/bin/robot ] ; then + python3 -mvenv ${venvroot} --upgrade + ${venvroot}/bin/pip3 install -q -r /srv/source/tests/pythonreq.txt +fi +cd /srv/source +mkdir -vp tests/outputs/ +exec ${venvroot}/bin/robot \ + -v NODE_ADDR:${LEADER_NODE} \ + -v IMAGE_URL:${IMAGE_URL} \ + -v SSH_KEYFILE:/tmp/idssh \ + -v RUN_UI_TESTS:${RUN_UI_TESTS} \ + --name test-ns8-module \ + --skiponfailure unstable \ + -d tests/outputs "${@}" tests/ +EOF diff --git a/scripts/tests/pythonreq.txt b/scripts/tests/pythonreq.txt new file mode 100644 index 0000000..279c2b3 --- /dev/null +++ b/scripts/tests/pythonreq.txt @@ -0,0 +1,3 @@ +robotframework +robotframework-sshlibrary +robotframework-browser From b101e19c4ef7c58eef8b135c51efef393cb9c3df Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Fri, 10 Apr 2026 15:13:04 +0200 Subject: [PATCH 02/27] fix: filter ui test cases if needed --- .github/workflows/test-module.yml | 1 + scripts/test-module.sh | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/test-module.yml b/.github/workflows/test-module.yml index 112fcf6..6e8a837 100644 --- a/.github/workflows/test-module.yml +++ b/.github/workflows/test-module.yml @@ -49,6 +49,7 @@ jobs: else echo "run_ui_tests=false" >> "$GITHUB_OUTPUT" fi + echo "//// run_ui_tests=$(grep run_ui_tests "$GITHUB_OUTPUT" | cut -d= -f2)" run_tests: needs: [module, chooser, detect_changes] diff --git a/scripts/test-module.sh b/scripts/test-module.sh index f901c10..37aa0b1 100644 --- a/scripts/test-module.sh +++ b/scripts/test-module.sh @@ -37,6 +37,14 @@ if [ ! -x ${venvroot}/bin/robot ] ; then fi cd /srv/source mkdir -vp tests/outputs/ + +# Exclude UI tests if RUN_UI_TESTS is not set to "true" +if [ "${RUN_UI_TESTS}" = "true" ]; then + ui_tag_filter="" +else + ui_tag_filter="--exclude ui" +fi + exec ${venvroot}/bin/robot \ -v NODE_ADDR:${LEADER_NODE} \ -v IMAGE_URL:${IMAGE_URL} \ @@ -44,5 +52,6 @@ exec ${venvroot}/bin/robot \ -v RUN_UI_TESTS:${RUN_UI_TESTS} \ --name test-ns8-module \ --skiponfailure unstable \ + ${ui_tag_filter} \ -d tests/outputs "${@}" tests/ EOF From 517f4660e16d646a957b4f015c652a79fc288849 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Fri, 10 Apr 2026 15:33:17 +0200 Subject: [PATCH 03/27] fix --- .github/workflows/module-info.yml | 1 + .github/workflows/test-module.yml | 2 ++ scripts/test-module.sh | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/module-info.yml b/.github/workflows/module-info.yml index 99358a2..eb97f81 100644 --- a/.github/workflows/module-info.yml +++ b/.github/workflows/module-info.yml @@ -31,6 +31,7 @@ on: jobs: module: + name: "Get module info" runs-on: ubuntu-latest outputs: owner: ${{ steps.owner.outputs.name }} diff --git a/.github/workflows/test-module.yml b/.github/workflows/test-module.yml index 6e8a837..8df9329 100644 --- a/.github/workflows/test-module.yml +++ b/.github/workflows/test-module.yml @@ -18,6 +18,7 @@ jobs: uses: ./.github/workflows/module-info.yml chooser: + name: "Choose nodes" runs-on: ubuntu-latest outputs: node_a: ${{ steps.pick.outputs.node_a }} @@ -34,6 +35,7 @@ jobs: fi detect_changes: + name: "Detect UI changes" runs-on: ubuntu-latest outputs: run_ui_tests: ${{ steps.changes.outputs.run_ui_tests }} diff --git a/scripts/test-module.sh b/scripts/test-module.sh index 37aa0b1..8084d32 100644 --- a/scripts/test-module.sh +++ b/scripts/test-module.sh @@ -26,7 +26,6 @@ podman run -i \ --env=venvroot \ --env=LEADER_NODE \ --env=IMAGE_URL \ - --env=RUN_UI_TESTS \ docker.io/python:3.11-alpine \ ash -l -s -- "${@}" <<'EOF' set -e @@ -34,6 +33,7 @@ echo "$ssh_key" > /tmp/idssh if [ ! -x ${venvroot}/bin/robot ] ; then python3 -mvenv ${venvroot} --upgrade ${venvroot}/bin/pip3 install -q -r /srv/source/tests/pythonreq.txt + ${venvroot}/bin/rfbrowser init fi cd /srv/source mkdir -vp tests/outputs/ From 9c5eb42c924bed817f66b756753f0c7ab981705d Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Fri, 10 Apr 2026 15:46:39 +0200 Subject: [PATCH 04/27] fix --- scripts/test-module.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-module.sh b/scripts/test-module.sh index 8084d32..bb6489d 100644 --- a/scripts/test-module.sh +++ b/scripts/test-module.sh @@ -33,7 +33,7 @@ echo "$ssh_key" > /tmp/idssh if [ ! -x ${venvroot}/bin/robot ] ; then python3 -mvenv ${venvroot} --upgrade ${venvroot}/bin/pip3 install -q -r /srv/source/tests/pythonreq.txt - ${venvroot}/bin/rfbrowser init + ${venvroot}/bin/python3 -m Browser.entry init fi cd /srv/source mkdir -vp tests/outputs/ From 7e87218de97e605fa2df58f139fbd52353a740c8 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Fri, 10 Apr 2026 16:14:38 +0200 Subject: [PATCH 05/27] Use different container images wrt ui tests --- scripts/test-module.sh | 23 +++++++++++++++++++---- scripts/tests/pythonreq-ui.txt | 2 ++ scripts/tests/pythonreq.txt | 1 - 3 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 scripts/tests/pythonreq-ui.txt diff --git a/scripts/test-module.sh b/scripts/test-module.sh index bb6489d..ce08464 100644 --- a/scripts/test-module.sh +++ b/scripts/test-module.sh @@ -18,6 +18,16 @@ venvroot=/usr/local/venv echo "Test! RUN_UI_TESTS=${RUN_UI_TESTS} ////" +if [ "${RUN_UI_TESTS}" = "true" ]; then + container_image="mcr.microsoft.com/playwright/python:v1.51.0-noble" + container_shell="bash" + pythonreq="/srv/source/tests/pythonreq-ui.txt" +else + container_image="docker.io/python:3.11-alpine" + container_shell="ash" + pythonreq="/srv/source/tests/pythonreq.txt" +fi + podman run -i \ --volume=.:/srv/source:z \ --volume=rftest-cache:${venvroot}:z \ @@ -26,14 +36,19 @@ podman run -i \ --env=venvroot \ --env=LEADER_NODE \ --env=IMAGE_URL \ - docker.io/python:3.11-alpine \ - ash -l -s -- "${@}" <<'EOF' + --env=RUN_UI_TESTS \ + --env=pythonreq \ + "${container_image}" \ + ${container_shell} -l -s -- "${@}" <<'EOF' set -e echo "$ssh_key" > /tmp/idssh if [ ! -x ${venvroot}/bin/robot ] ; then python3 -mvenv ${venvroot} --upgrade - ${venvroot}/bin/pip3 install -q -r /srv/source/tests/pythonreq.txt - ${venvroot}/bin/python3 -m Browser.entry init + ${venvroot}/bin/pip3 install -q -r ${pythonreq} +fi +if [ "${RUN_UI_TESTS}" = "true" ] && [ ! -f ${venvroot}/.rfbrowser_initialized ] ; then + ${venvroot}/bin/rfbrowser init + touch ${venvroot}/.rfbrowser_initialized fi cd /srv/source mkdir -vp tests/outputs/ diff --git a/scripts/tests/pythonreq-ui.txt b/scripts/tests/pythonreq-ui.txt new file mode 100644 index 0000000..4cd8c1f --- /dev/null +++ b/scripts/tests/pythonreq-ui.txt @@ -0,0 +1,2 @@ +-r pythonreq.txt +robotframework-browser diff --git a/scripts/tests/pythonreq.txt b/scripts/tests/pythonreq.txt index 279c2b3..af1d9bf 100644 --- a/scripts/tests/pythonreq.txt +++ b/scripts/tests/pythonreq.txt @@ -1,3 +1,2 @@ robotframework robotframework-sshlibrary -robotframework-browser From 148ecee75b3ee285edfdfec3fdc07c780db28897 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Fri, 10 Apr 2026 16:46:10 +0200 Subject: [PATCH 06/27] fix --- scripts/test-module.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/test-module.sh b/scripts/test-module.sh index ce08464..781bc5b 100644 --- a/scripts/test-module.sh +++ b/scripts/test-module.sh @@ -43,6 +43,9 @@ podman run -i \ set -e echo "$ssh_key" > /tmp/idssh if [ ! -x ${venvroot}/bin/robot ] ; then + if command -v apt-get > /dev/null 2>&1; then + apt-get install -y -q python3-venv + fi python3 -mvenv ${venvroot} --upgrade ${venvroot}/bin/pip3 install -q -r ${pythonreq} fi From 91c98e86a6dd80325c23a6cb06a6a67d4e732191 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Fri, 10 Apr 2026 16:56:12 +0200 Subject: [PATCH 07/27] fix --- scripts/test-module.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test-module.sh b/scripts/test-module.sh index 781bc5b..3cce18c 100644 --- a/scripts/test-module.sh +++ b/scripts/test-module.sh @@ -44,7 +44,7 @@ set -e echo "$ssh_key" > /tmp/idssh if [ ! -x ${venvroot}/bin/robot ] ; then if command -v apt-get > /dev/null 2>&1; then - apt-get install -y -q python3-venv + apt install -y -q python3.12-venv fi python3 -mvenv ${venvroot} --upgrade ${venvroot}/bin/pip3 install -q -r ${pythonreq} From f20fe08a8dfc52208d2962eff45a6702b6d8da55 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Fri, 10 Apr 2026 17:04:11 +0200 Subject: [PATCH 08/27] fix --- scripts/test-module.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/test-module.sh b/scripts/test-module.sh index 3cce18c..97a2934 100644 --- a/scripts/test-module.sh +++ b/scripts/test-module.sh @@ -13,23 +13,25 @@ LEADER_NODE="${1:?missing LEADER_NODE argument}" IMAGE_URL="${2:?missing IMAGE_URL argument}" shift 2 -ssh_key="$(< $SSH_KEYFILE)" +ssh_key="$(<$SSH_KEYFILE)" venvroot=/usr/local/venv +script_dir="$(cd "$(dirname "$0")" && pwd)" echo "Test! RUN_UI_TESTS=${RUN_UI_TESTS} ////" if [ "${RUN_UI_TESTS}" = "true" ]; then - container_image="mcr.microsoft.com/playwright/python:v1.51.0-noble" + container_image="mcr.microsoft.com/playwright:v1.51.0-noble" container_shell="bash" - pythonreq="/srv/source/tests/pythonreq-ui.txt" + pythonreq="/srv/ns8-github-actions/tests/pythonreq-ui.txt" else container_image="docker.io/python:3.11-alpine" container_shell="ash" - pythonreq="/srv/source/tests/pythonreq.txt" + pythonreq="/srv/ns8-github-actions/tests/pythonreq.txt" fi podman run -i \ --volume=.:/srv/source:z \ + --volume=${script_dir}/tests:/srv/ns8-github-actions/tests:z \ --volume=rftest-cache:${venvroot}:z \ --replace --name=rftest \ --env=ssh_key \ @@ -44,9 +46,13 @@ set -e echo "$ssh_key" > /tmp/idssh if [ ! -x ${venvroot}/bin/robot ] ; then if command -v apt-get > /dev/null 2>&1; then - apt install -y -q python3.12-venv + # mcr.microsoft.com/playwright:*-noble has npm pre-installed but no Python + apt-get update -q + apt-get install -y -q python3 python3-venv + python3 -mvenv ${venvroot} + else + python3 -mvenv ${venvroot} --upgrade fi - python3 -mvenv ${venvroot} --upgrade ${venvroot}/bin/pip3 install -q -r ${pythonreq} fi if [ "${RUN_UI_TESTS}" = "true" ] && [ ! -f ${venvroot}/.rfbrowser_initialized ] ; then From 860adc6ac970bcba90f3698a04ac95d9156d073c Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Mon, 13 Apr 2026 10:51:51 +0200 Subject: [PATCH 09/27] add screenshots to pull request --- .../workflows/test-on-digitalocean-infra.yml | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index cc4154d..ac940ab 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -243,6 +243,66 @@ jobs: -f "target_url=$status_page" \ -f "description=Test on DigitalOcean (${{ matrix.node }})" \ -f "context=continuous-integration/robotframework/${{ matrix.node }})" + - name: "Comment PR with UI test screenshots" + if: inputs.run_ui_tests == true + uses: actions/github-script@v7 + env: + MODULE_PATH: ${{ inputs.path }} + MATRIX_NODE: ${{ matrix.node }} + with: + script: | + const fs = require('fs'); + const path = require('path'); + + const prNumber = context.payload.pull_request?.number + ?? context.payload.workflow_run?.pull_requests?.[0]?.number; + if (!prNumber) { + core.notice('Not triggered from a pull request, skipping screenshot comment'); + return; + } + + const modulePath = process.env.MODULE_PATH || ''; + const matrixNode = process.env.MATRIX_NODE; + const outputsDir = path.join( + process.env.GITHUB_WORKSPACE, + 'module', + ...modulePath ? [modulePath] : [], + 'tests', 'outputs' + ); + + const screenshots = []; + function findScreenshots(dir) { + if (!fs.existsSync(dir)) return; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + findScreenshots(fullPath); + } else if (/\.(png|jpg|jpeg|gif|webp)$/i.test(entry.name)) { + screenshots.push(path.relative(outputsDir, fullPath)); + } + } + } + findScreenshots(outputsDir); + + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const artifactName = `tests-logs-${matrixNode}`; + + let body = `### UI Test Screenshots \`${matrixNode}\`\n\n`; + if (screenshots.length > 0) { + body += `${screenshots.length} screenshot(s) captured. Download the \`${artifactName}\` artifact from the [workflow run](${runUrl}):\n\n`; + for (const s of screenshots) { + body += `- \`${s}\`\n`; + } + } else { + body += `No screenshots found in test outputs. See the [workflow run](${runUrl}) for full artifacts.\n`; + } + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body, + }); - name: "Debugging with tmate" if: ${{ inputs.debug_shell == true && ! cancelled() && always() }} uses: mxschmitt/action-tmate@v3 From 83f3f7f937724daae0ea93590562fd2b98fde9b2 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Mon, 13 Apr 2026 12:51:44 +0200 Subject: [PATCH 10/27] fix --- .../workflows/test-on-digitalocean-infra.yml | 19 +++++--- scripts/test-module.sh | 47 ++++++++++++++++--- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index ac940ab..aedf6c7 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -254,11 +254,18 @@ jobs: const fs = require('fs'); const path = require('path'); + //// TODO: remove fallback PR once PR detection is confirmed to work + const fallbackOwner = 'NethServer'; + const fallbackRepo = 'ns8-mail'; + const fallbackPrNumber = 252; + const prNumber = context.payload.pull_request?.number - ?? context.payload.workflow_run?.pull_requests?.[0]?.number; - if (!prNumber) { - core.notice('Not triggered from a pull request, skipping screenshot comment'); - return; + ?? context.payload.workflow_run?.pull_requests?.[0]?.number + ?? fallbackPrNumber; + const prOwner = prNumber === fallbackPrNumber ? fallbackOwner : context.repo.owner; + const prRepo = prNumber === fallbackPrNumber ? fallbackRepo : context.repo.repo; + if (prNumber === fallbackPrNumber) { + core.warning(`PR number not detected from context, falling back to ${fallbackOwner}/${fallbackRepo}#${fallbackPrNumber} for testing`); } const modulePath = process.env.MODULE_PATH || ''; @@ -298,8 +305,8 @@ jobs: } await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, + owner: prOwner, + repo: prRepo, issue_number: prNumber, body, }); diff --git a/scripts/test-module.sh b/scripts/test-module.sh index 97a2934..acdd021 100644 --- a/scripts/test-module.sh +++ b/scripts/test-module.sh @@ -7,18 +7,28 @@ set -e -a +# Path to the SSH private key used to connect to the NS8 leader node SSH_KEYFILE=${SSH_KEYFILE:-$HOME/.ssh/id_rsa} +# Mandatory positional arguments: the leader node hostname and the module image URL LEADER_NODE="${1:?missing LEADER_NODE argument}" IMAGE_URL="${2:?missing IMAGE_URL argument}" shift 2 +# Read SSH key contents so they can be injected into the container via an env var ssh_key="$(<$SSH_KEYFILE)" + +# The venv is stored in a named volume (rftest-cache) to cache pip/rfbrowser installs across runs venvroot=/usr/local/venv + +# Resolve the directory of this script to mount the shared test requirements inside the container script_dir="$(cd "$(dirname "$0")" && pwd)" echo "Test! RUN_UI_TESTS=${RUN_UI_TESTS} ////" +# Select the container image and Python requirements file based on whether UI tests are enabled. +# UI tests require the Playwright image (Debian-based, includes browser binaries). +# Non-UI tests use a lightweight Alpine Python image. if [ "${RUN_UI_TESTS}" = "true" ]; then container_image="mcr.microsoft.com/playwright:v1.51.0-noble" container_shell="bash" @@ -29,6 +39,12 @@ else pythonreq="/srv/ns8-github-actions/tests/pythonreq.txt" fi +# Run the test suite inside a container. +# Mounts: +# . → /srv/source (module source tree, including the tests/ directory) +# scripts/tests → /srv/ns8-github-actions/tests (shared Robot Framework helpers and requirements) +# rftest-cache → ${venvroot} (named volume to persist the Python venv across runs) +# Any extra arguments after IMAGE_URL are forwarded to the robot command inside the container. podman run -i \ --volume=.:/srv/source:z \ --volume=${script_dir}/tests:/srv/ns8-github-actions/tests:z \ @@ -43,22 +59,41 @@ podman run -i \ "${container_image}" \ ${container_shell} -l -s -- "${@}" <<'EOF' set -e + +# Write the SSH private key to a temp file for use by Robot Framework tests echo "$ssh_key" > /tmp/idssh -if [ ! -x ${venvroot}/bin/robot ] ; then + +# Install the Python venv and Robot Framework dependencies if not already cached. +# Cache is invalidated when the MD5 checksum of the requirements file changes, ensuring +# that dependency updates are always picked up even on reused (self-hosted) runners. +pythonreq_checksum_file="${venvroot}/.pythonreq.md5" +pythonreq_current_checksum=$(md5sum "${pythonreq}" | cut -d' ' -f1) +pythonreq_cached_checksum=$(cat "${pythonreq_checksum_file}" 2>/dev/null || true) + +if [ ! -x "${venvroot}/bin/robot" ] || [ "${pythonreq_current_checksum}" != "${pythonreq_cached_checksum}" ] ; then if command -v apt-get > /dev/null 2>&1; then # mcr.microsoft.com/playwright:*-noble has npm pre-installed but no Python apt-get update -q apt-get install -y -q python3 python3-venv - python3 -mvenv ${venvroot} + python3 -mvenv "${venvroot}" else - python3 -mvenv ${venvroot} --upgrade + # Alpine image already has Python; --upgrade refreshes pip/setuptools in-place + python3 -mvenv "${venvroot}" --upgrade fi - ${venvroot}/bin/pip3 install -q -r ${pythonreq} + ${venvroot}/bin/pip3 install -q -r "${pythonreq}" + # Save the checksum so future runs can detect requirement changes + echo "${pythonreq_current_checksum}" > "${pythonreq_checksum_file}" + # Invalidate the rfbrowser sentinel so it is re-initialized with the new packages + rm -f "${venvroot}/.rfbrowser_initialized" fi -if [ "${RUN_UI_TESTS}" = "true" ] && [ ! -f ${venvroot}/.rfbrowser_initialized ] ; then + +# Initialize the Playwright browser binaries for rfbrowser (UI tests only). +# A sentinel file prevents re-running this expensive step on cache hits. +if [ "${RUN_UI_TESTS}" = "true" ] && [ ! -f "${venvroot}/.rfbrowser_initialized" ] ; then ${venvroot}/bin/rfbrowser init - touch ${venvroot}/.rfbrowser_initialized + touch "${venvroot}/.rfbrowser_initialized" fi + cd /srv/source mkdir -vp tests/outputs/ From 4c32fba548247a82e2e5b5c817a7c8ad3242a1ed Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Mon, 13 Apr 2026 12:53:52 +0200 Subject: [PATCH 11/27] fix --- .github/workflows/test-on-digitalocean-infra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index aedf6c7..5203d31 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -245,7 +245,7 @@ jobs: -f "context=continuous-integration/robotframework/${{ matrix.node }})" - name: "Comment PR with UI test screenshots" if: inputs.run_ui_tests == true - uses: actions/github-script@v7 + uses: actions/github-script@v9 env: MODULE_PATH: ${{ inputs.path }} MATRIX_NODE: ${{ matrix.node }} From a45f4708e0bf3762cf801cf0ac08058be0b90a15 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Mon, 13 Apr 2026 14:28:24 +0200 Subject: [PATCH 12/27] fix --- .../workflows/test-on-digitalocean-infra.yml | 83 +++++-------------- 1 file changed, 19 insertions(+), 64 deletions(-) diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index 5203d31..bbd82d2 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -243,73 +243,28 @@ jobs: -f "target_url=$status_page" \ -f "description=Test on DigitalOcean (${{ matrix.node }})" \ -f "context=continuous-integration/robotframework/${{ matrix.node }})" - - name: "Comment PR with UI test screenshots" + - name: "Setup CML" + if: inputs.run_ui_tests == true + uses: iterative/setup-cml@v2 + with: + vega: false + - name: "Post screenshots to PR" if: inputs.run_ui_tests == true - uses: actions/github-script@v9 env: + REPO_TOKEN: ${{ github.token }} MODULE_PATH: ${{ inputs.path }} - MATRIX_NODE: ${{ matrix.node }} - with: - script: | - const fs = require('fs'); - const path = require('path'); - - //// TODO: remove fallback PR once PR detection is confirmed to work - const fallbackOwner = 'NethServer'; - const fallbackRepo = 'ns8-mail'; - const fallbackPrNumber = 252; - - const prNumber = context.payload.pull_request?.number - ?? context.payload.workflow_run?.pull_requests?.[0]?.number - ?? fallbackPrNumber; - const prOwner = prNumber === fallbackPrNumber ? fallbackOwner : context.repo.owner; - const prRepo = prNumber === fallbackPrNumber ? fallbackRepo : context.repo.repo; - if (prNumber === fallbackPrNumber) { - core.warning(`PR number not detected from context, falling back to ${fallbackOwner}/${fallbackRepo}#${fallbackPrNumber} for testing`); - } - - const modulePath = process.env.MODULE_PATH || ''; - const matrixNode = process.env.MATRIX_NODE; - const outputsDir = path.join( - process.env.GITHUB_WORKSPACE, - 'module', - ...modulePath ? [modulePath] : [], - 'tests', 'outputs' - ); - - const screenshots = []; - function findScreenshots(dir) { - if (!fs.existsSync(dir)) return; - for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory()) { - findScreenshots(fullPath); - } else if (/\.(png|jpg|jpeg|gif|webp)$/i.test(entry.name)) { - screenshots.push(path.relative(outputsDir, fullPath)); - } - } - } - findScreenshots(outputsDir); - - const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const artifactName = `tests-logs-${matrixNode}`; - - let body = `### UI Test Screenshots \`${matrixNode}\`\n\n`; - if (screenshots.length > 0) { - body += `${screenshots.length} screenshot(s) captured. Download the \`${artifactName}\` artifact from the [workflow run](${runUrl}):\n\n`; - for (const s of screenshots) { - body += `- \`${s}\`\n`; - } - } else { - body += `No screenshots found in test outputs. See the [workflow run](${runUrl}) for full artifacts.\n`; - } - - await github.rest.issues.createComment({ - owner: prOwner, - repo: prRepo, - issue_number: prNumber, - body, - }); + run: | + output_dir="${{ github.workspace }}/module${MODULE_PATH:+/$MODULE_PATH}/tests/outputs" + echo "### UI Test Screenshots \`${{ matrix.node }}\`" >> comment.md + mapfile -d '' screenshots < <(find "$output_dir" -type f \( -iname '*.png' -o -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.gif' -o -iname '*.webp' \) -print0 2>/dev/null | sort -z) + if [ ${#screenshots[@]} -eq 0 ]; then + echo "No screenshots found in test outputs." >> comment.md + else + for f in "${screenshots[@]}"; do + cml publish --md --title "$(basename "$f")" "$f" >> comment.md + done + fi + cml comment create --target=pr/252 comment.md #### TODO: remove hardcoded PR - name: "Debugging with tmate" if: ${{ inputs.debug_shell == true && ! cancelled() && always() }} uses: mxschmitt/action-tmate@v3 From 5a460a0fa66104496cde498b26482f73bc1eac9d Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Mon, 13 Apr 2026 14:57:35 +0200 Subject: [PATCH 13/27] fix --- .github/workflows/test-on-digitalocean-infra.yml | 4 ++-- scripts/test-module.sh | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index bbd82d2..59f3f9e 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -261,10 +261,10 @@ jobs: echo "No screenshots found in test outputs." >> comment.md else for f in "${screenshots[@]}"; do - cml publish --md --title "$(basename "$f")" "$f" >> comment.md + cml publish --md --title "$(basename "$f")" "$f" --repo="${{ github.repository }}" >> comment.md done fi - cml comment create --target=pr/252 comment.md #### TODO: remove hardcoded PR + cml comment create --target=pr/252 --repo="${{ github.repository }}" comment.md #### //// TODO: remove hardcoded PR - name: "Debugging with tmate" if: ${{ inputs.debug_shell == true && ! cancelled() && always() }} uses: mxschmitt/action-tmate@v3 diff --git a/scripts/test-module.sh b/scripts/test-module.sh index acdd021..f53e513 100644 --- a/scripts/test-module.sh +++ b/scripts/test-module.sh @@ -7,6 +7,9 @@ set -e -a +# //// +_script_start=$(date +%s) + # Path to the SSH private key used to connect to the NS8 leader node SSH_KEYFILE=${SSH_KEYFILE:-$HOME/.ssh/id_rsa} @@ -56,6 +59,7 @@ podman run -i \ --env=IMAGE_URL \ --env=RUN_UI_TESTS \ --env=pythonreq \ + --env=_script_start \ "${container_image}" \ ${container_shell} -l -s -- "${@}" <<'EOF' set -e @@ -104,6 +108,8 @@ else ui_tag_filter="--exclude ui" fi +echo "DEBUG: $(( $(date +%s) - _script_start ))s elapsed from script start to robot launch ////" + exec ${venvroot}/bin/robot \ -v NODE_ADDR:${LEADER_NODE} \ -v IMAGE_URL:${IMAGE_URL} \ From 600fcd0485a3a5ad7a56cd94e0a0a2ae337e3c7e Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Mon, 13 Apr 2026 16:54:14 +0200 Subject: [PATCH 14/27] fix --- .github/workflows/test-on-digitalocean-infra.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index 59f3f9e..a10f617 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -261,10 +261,10 @@ jobs: echo "No screenshots found in test outputs." >> comment.md else for f in "${screenshots[@]}"; do - cml publish --md --title "$(basename "$f")" "$f" --repo="${{ github.repository }}" >> comment.md + cml publish --md --title "$(basename "$f")" "$f" --repo="https://github.com/${{ github.repository }}" >> comment.md done fi - cml comment create --target=pr/252 --repo="${{ github.repository }}" comment.md #### //// TODO: remove hardcoded PR + cml comment create --target=pr/252 --repo="https://github.com/${{ github.repository }}" comment.md #### //// TODO: remove hardcoded PR - name: "Debugging with tmate" if: ${{ inputs.debug_shell == true && ! cancelled() && always() }} uses: mxschmitt/action-tmate@v3 From 055d05538145a43fe84b5813224a110547665a7d Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Mon, 13 Apr 2026 17:38:45 +0200 Subject: [PATCH 15/27] use @dvcorg/cml instead of cml github action --- .github/workflows/test-on-digitalocean-infra.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index a10f617..1637d8a 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -245,9 +245,7 @@ jobs: -f "context=continuous-integration/robotframework/${{ matrix.node }})" - name: "Setup CML" if: inputs.run_ui_tests == true - uses: iterative/setup-cml@v2 - with: - vega: false + run: npm install --location=global @dvcorg/cml - name: "Post screenshots to PR" if: inputs.run_ui_tests == true env: From 4e28f71ad2904d2666e9cf0e309f11294175744a Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Mon, 13 Apr 2026 17:41:26 +0200 Subject: [PATCH 16/27] avoid using cml publish --- .github/workflows/test-on-digitalocean-infra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index 1637d8a..4809f3e 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -259,7 +259,7 @@ jobs: echo "No screenshots found in test outputs." >> comment.md else for f in "${screenshots[@]}"; do - cml publish --md --title "$(basename "$f")" "$f" --repo="https://github.com/${{ github.repository }}" >> comment.md + echo "![$(basename "$f")]($f)" >> comment.md done fi cml comment create --target=pr/252 --repo="https://github.com/${{ github.repository }}" comment.md #### //// TODO: remove hardcoded PR From 4b378fe3284c7e05a58ec52cb0bef129b972dc23 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Mon, 13 Apr 2026 17:59:51 +0200 Subject: [PATCH 17/27] always run ui tests --- .github/workflows/test-module.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-module.yml b/.github/workflows/test-module.yml index 8df9329..e89b0c6 100644 --- a/.github/workflows/test-module.yml +++ b/.github/workflows/test-module.yml @@ -70,7 +70,8 @@ jobs: }} args: ${{ format('ghcr.io/{0}/{1}:{2} -v SCENARIO:{3}', needs.module.outputs.owner, needs.module.outputs.name, needs.module.outputs.tag, matrix.scenario) }} repo_ref: ${{ needs.module.outputs.sha }} - run_ui_tests: ${{ needs.detect_changes.outputs.run_ui_tests == 'true' }} + run_ui_tests: true + # //// run_ui_tests: ${{ needs.detect_changes.outputs.run_ui_tests == 'true' }} debug_shell: ${{ inputs.debug_shell }} secrets: do_token: ${{ secrets.do_token }} From ad50b805e09581012e6da15739ac16279bab172b Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Tue, 14 Apr 2026 08:41:50 +0200 Subject: [PATCH 18/27] avoid using cml publish --- scripts/test-module.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/test-module.sh b/scripts/test-module.sh index f53e513..5fd4937 100644 --- a/scripts/test-module.sh +++ b/scripts/test-module.sh @@ -33,7 +33,7 @@ echo "Test! RUN_UI_TESTS=${RUN_UI_TESTS} ////" # UI tests require the Playwright image (Debian-based, includes browser binaries). # Non-UI tests use a lightweight Alpine Python image. if [ "${RUN_UI_TESTS}" = "true" ]; then - container_image="mcr.microsoft.com/playwright:v1.51.0-noble" + container_image="mcr.microsoft.com/playwright/python:v1.51.0-noble" container_shell="bash" pythonreq="/srv/ns8-github-actions/tests/pythonreq-ui.txt" else @@ -76,9 +76,9 @@ pythonreq_cached_checksum=$(cat "${pythonreq_checksum_file}" 2>/dev/null || true if [ ! -x "${venvroot}/bin/robot" ] || [ "${pythonreq_current_checksum}" != "${pythonreq_cached_checksum}" ] ; then if command -v apt-get > /dev/null 2>&1; then - # mcr.microsoft.com/playwright:*-noble has npm pre-installed but no Python + # mcr.microsoft.com/playwright/python:*-noble has Python but not venv apt-get update -q - apt-get install -y -q python3 python3-venv + apt-get install -y -q python3-venv python3 -mvenv "${venvroot}" else # Alpine image already has Python; --upgrade refreshes pip/setuptools in-place From 3d833353c12c23b4c8f3ed8462ca99c3db52e373 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Tue, 14 Apr 2026 09:19:31 +0200 Subject: [PATCH 19/27] avoid using cml publish --- scripts/test-module.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/test-module.sh b/scripts/test-module.sh index 5fd4937..f53e513 100644 --- a/scripts/test-module.sh +++ b/scripts/test-module.sh @@ -33,7 +33,7 @@ echo "Test! RUN_UI_TESTS=${RUN_UI_TESTS} ////" # UI tests require the Playwright image (Debian-based, includes browser binaries). # Non-UI tests use a lightweight Alpine Python image. if [ "${RUN_UI_TESTS}" = "true" ]; then - container_image="mcr.microsoft.com/playwright/python:v1.51.0-noble" + container_image="mcr.microsoft.com/playwright:v1.51.0-noble" container_shell="bash" pythonreq="/srv/ns8-github-actions/tests/pythonreq-ui.txt" else @@ -76,9 +76,9 @@ pythonreq_cached_checksum=$(cat "${pythonreq_checksum_file}" 2>/dev/null || true if [ ! -x "${venvroot}/bin/robot" ] || [ "${pythonreq_current_checksum}" != "${pythonreq_cached_checksum}" ] ; then if command -v apt-get > /dev/null 2>&1; then - # mcr.microsoft.com/playwright/python:*-noble has Python but not venv + # mcr.microsoft.com/playwright:*-noble has npm pre-installed but no Python apt-get update -q - apt-get install -y -q python3-venv + apt-get install -y -q python3 python3-venv python3 -mvenv "${venvroot}" else # Alpine image already has Python; --upgrade refreshes pip/setuptools in-place From 358cf4ac232ceed6e21b89c55043a8630c2c5203 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Tue, 14 Apr 2026 09:57:14 +0200 Subject: [PATCH 20/27] fix --- .github/workflows/test-module.yml | 21 ++++++++++++------- .../workflows/test-on-digitalocean-infra.yml | 1 + 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-module.yml b/.github/workflows/test-module.yml index e89b0c6..da67bc8 100644 --- a/.github/workflows/test-module.yml +++ b/.github/workflows/test-module.yml @@ -3,6 +3,11 @@ name: Test module on: workflow_call: inputs: + run_ui_tests: + description: "When to run UI tests: on_ui_change, on_renovate_ui_change, never" + required: false + type: string + default: on_renovate_ui_change debug_shell: description: "Debug shell" required: false @@ -34,11 +39,11 @@ jobs: echo "node_b=rl1" >> "$GITHUB_OUTPUT" fi - detect_changes: + detect_ui_changes: name: "Detect UI changes" runs-on: ubuntu-latest outputs: - run_ui_tests: ${{ steps.changes.outputs.run_ui_tests }} + ui_has_changed: ${{ steps.changes.outputs.ui_has_changed }} steps: - uses: actions/checkout@v6 with: @@ -47,14 +52,14 @@ jobs: - id: changes run: | if git diff --name-only HEAD^ HEAD | grep -qE '^ui/|^build-images\.sh$'; then - echo "run_ui_tests=true" >> "$GITHUB_OUTPUT" + echo "ui_has_changed=true" >> "$GITHUB_OUTPUT" else - echo "run_ui_tests=false" >> "$GITHUB_OUTPUT" + echo "ui_has_changed=false" >> "$GITHUB_OUTPUT" fi - echo "//// run_ui_tests=$(grep run_ui_tests "$GITHUB_OUTPUT" | cut -d= -f2)" + echo "//// ui_has_changed=$(grep ui_has_changed "$GITHUB_OUTPUT" | cut -d= -f2)" run_tests: - needs: [module, chooser, detect_changes] + needs: [module, chooser, detect_ui_changes] strategy: fail-fast: false matrix: @@ -70,8 +75,8 @@ jobs: }} args: ${{ format('ghcr.io/{0}/{1}:{2} -v SCENARIO:{3}', needs.module.outputs.owner, needs.module.outputs.name, needs.module.outputs.tag, matrix.scenario) }} repo_ref: ${{ needs.module.outputs.sha }} - run_ui_tests: true - # //// run_ui_tests: ${{ needs.detect_changes.outputs.run_ui_tests == 'true' }} + # run_ui_tests: true //// + run_ui_tests: ${{ inputs.run_ui_tests != 'never' && needs.detect_ui_changes.outputs.ui_has_changed == 'true' }} debug_shell: ${{ inputs.debug_shell }} secrets: do_token: ${{ secrets.do_token }} diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index 4809f3e..9da73ae 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -259,6 +259,7 @@ jobs: echo "No screenshots found in test outputs." >> comment.md else for f in "${screenshots[@]}"; do + echo "**$(basename "$f")**" >> comment.md echo "![$(basename "$f")]($f)" >> comment.md done fi From 4509289c090c26ab5ae7647be4909b3c20b89981 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Tue, 14 Apr 2026 09:59:10 +0200 Subject: [PATCH 21/27] fix --- .github/workflows/test-module.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-module.yml b/.github/workflows/test-module.yml index da67bc8..f686a1e 100644 --- a/.github/workflows/test-module.yml +++ b/.github/workflows/test-module.yml @@ -75,8 +75,8 @@ jobs: }} args: ${{ format('ghcr.io/{0}/{1}:{2} -v SCENARIO:{3}', needs.module.outputs.owner, needs.module.outputs.name, needs.module.outputs.tag, matrix.scenario) }} repo_ref: ${{ needs.module.outputs.sha }} - # run_ui_tests: true //// - run_ui_tests: ${{ inputs.run_ui_tests != 'never' && needs.detect_ui_changes.outputs.ui_has_changed == 'true' }} + run_ui_tests: true + # //// run_ui_tests: ${{ inputs.run_ui_tests != 'never' && needs.detect_ui_changes.outputs.ui_has_changed == 'true' }} debug_shell: ${{ inputs.debug_shell }} secrets: do_token: ${{ secrets.do_token }} From b632709712905b61bd1a5d9a65b3f2f5d841645b Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Tue, 14 Apr 2026 10:54:47 +0200 Subject: [PATCH 22/27] fix --- .github/workflows/test-on-digitalocean-infra.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index 9da73ae..caa7500 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -259,7 +259,8 @@ jobs: echo "No screenshots found in test outputs." >> comment.md else for f in "${screenshots[@]}"; do - echo "**$(basename "$f")**" >> comment.md + name=$(basename "${f%.*}") + echo "**${name^}**" >> comment.md echo "![$(basename "$f")]($f)" >> comment.md done fi From 16c00edd6fcab5f5a7d2d371e1aa26c0ca701fca Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Tue, 14 Apr 2026 10:56:57 +0200 Subject: [PATCH 23/27] fix --- .github/workflows/test-on-digitalocean-infra.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index caa7500..4f53725 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -261,7 +261,9 @@ jobs: for f in "${screenshots[@]}"; do name=$(basename "${f%.*}") echo "**${name^}**" >> comment.md + echo "" >> comment.md echo "![$(basename "$f")]($f)" >> comment.md + echo "" >> comment.md done fi cml comment create --target=pr/252 --repo="https://github.com/${{ github.repository }}" comment.md #### //// TODO: remove hardcoded PR From e1fedae4dd4cef3c0760fd25b7bad8f649d2e699 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Tue, 14 Apr 2026 15:57:53 +0200 Subject: [PATCH 24/27] improve detect changes job --- .github/workflows/test-module.yml | 44 ++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test-module.yml b/.github/workflows/test-module.yml index f686a1e..ee3ff6e 100644 --- a/.github/workflows/test-module.yml +++ b/.github/workflows/test-module.yml @@ -3,7 +3,7 @@ name: Test module on: workflow_call: inputs: - run_ui_tests: + ui_tests_strategy: description: "When to run UI tests: on_ui_change, on_renovate_ui_change, never" required: false type: string @@ -39,11 +39,11 @@ jobs: echo "node_b=rl1" >> "$GITHUB_OUTPUT" fi - detect_ui_changes: - name: "Detect UI changes" + detect_changes: + name: "Detect changes" runs-on: ubuntu-latest outputs: - ui_has_changed: ${{ steps.changes.outputs.ui_has_changed }} + should_run_ui_tests: ${{ steps.changes.outputs.should_run_ui_tests }} steps: - uses: actions/checkout@v6 with: @@ -52,14 +52,39 @@ jobs: - id: changes run: | if git diff --name-only HEAD^ HEAD | grep -qE '^ui/|^build-images\.sh$'; then - echo "ui_has_changed=true" >> "$GITHUB_OUTPUT" + has_ui_changes=true else - echo "ui_has_changed=false" >> "$GITHUB_OUTPUT" + has_ui_changes=false fi - echo "//// ui_has_changed=$(grep ui_has_changed "$GITHUB_OUTPUT" | cut -d= -f2)" + + branch_name="${{ github.event.workflow_run.head_branch || github.ref_name }}" + if [[ "$branch_name" == renovate-* || "$branch_name" == renovate/* ]]; then + is_renovate=true + else + is_renovate=false + fi + + ui_tests_strategy="${{ inputs.ui_tests_strategy }}" + if [[ "$ui_tests_strategy" == "never" ]]; then + should_run_ui_tests=false + elif [[ "$ui_tests_strategy" == "on_ui_change" ]]; then + should_run_ui_tests=$has_ui_changes + elif [[ "$ui_tests_strategy" == "on_renovate_ui_change" ]]; then + if [[ "$has_ui_changes" == "true" && "$is_renovate" == "true" ]]; then + should_run_ui_tests=true + else + should_run_ui_tests=false + fi + else + should_run_ui_tests=false + fi + + echo "should_run_ui_tests=$should_run_ui_tests" >> "$GITHUB_OUTPUT" + + echo "//// branch=$branch_name has_ui_changes=$has_ui_changes is_renovate=$is_renovate strategy=$ui_tests_strategy should_run_ui_tests=$should_run_ui_tests" run_tests: - needs: [module, chooser, detect_ui_changes] + needs: [module, chooser, detect_changes] strategy: fail-fast: false matrix: @@ -75,8 +100,7 @@ jobs: }} args: ${{ format('ghcr.io/{0}/{1}:{2} -v SCENARIO:{3}', needs.module.outputs.owner, needs.module.outputs.name, needs.module.outputs.tag, matrix.scenario) }} repo_ref: ${{ needs.module.outputs.sha }} - run_ui_tests: true - # //// run_ui_tests: ${{ inputs.run_ui_tests != 'never' && needs.detect_ui_changes.outputs.ui_has_changed == 'true' }} + run_ui_tests: ${{ needs.detect_changes.outputs.should_run_ui_tests == 'true' }} debug_shell: ${{ inputs.debug_shell }} secrets: do_token: ${{ secrets.do_token }} From 1c78f41b25caf94020e2feb2423aa111ed8b2780 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Tue, 14 Apr 2026 16:55:53 +0200 Subject: [PATCH 25/27] improve detect changes job --- .github/workflows/check-ui-tests-needed.yml | 76 +++++++++++++++++++++ .github/workflows/test-module.yml | 54 +++------------ 2 files changed, 85 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/check-ui-tests-needed.yml diff --git a/.github/workflows/check-ui-tests-needed.yml b/.github/workflows/check-ui-tests-needed.yml new file mode 100644 index 0000000..7d6e404 --- /dev/null +++ b/.github/workflows/check-ui-tests-needed.yml @@ -0,0 +1,76 @@ +name: Check UI tests needed + +on: + workflow_call: + inputs: + ui_tests_strategy: + description: "When to run UI tests: on_ui_change, on_renovate_ui_change, never" + required: false + type: string + default: on_renovate_ui_change + head_sha: + description: "The commit SHA to checkout" + required: true + type: string + head_branch: + description: "The branch name to check for renovate" + required: true + type: string + ui_dir: + description: "The UI directory to watch for changes" + required: false + type: string + default: ui + build_script: + description: "The build script file to watch for changes (changes to this file may affect the UI)" + required: false + type: string + default: build-images.sh + outputs: + ui_tests_needed: + description: "Whether UI tests should run based on strategy and detected changes" + value: ${{ jobs.detect_changes.outputs.ui_tests_needed }} + +jobs: + detect_changes: + name: "Detect changes" + runs-on: ubuntu-latest + outputs: + ui_tests_needed: ${{ steps.changes.outputs.ui_tests_needed }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.head_sha }} + fetch-depth: 2 + - id: changes + run: | + if git diff --name-only HEAD^ HEAD | grep -qE "^${{ inputs.ui_dir }}/|^${{ inputs.build_script }}$"; then + has_ui_changes=true + else + has_ui_changes=false + fi + + branch_name="${{ inputs.head_branch }}" + if [[ "$branch_name" == renovate-* || "$branch_name" == renovate/* ]]; then + is_renovate=true + else + is_renovate=false + fi + + ui_tests_strategy="${{ inputs.ui_tests_strategy }}" + if [[ "$ui_tests_strategy" == "never" ]]; then + should_run_ui_tests=false + elif [[ "$ui_tests_strategy" == "on_ui_change" ]]; then + should_run_ui_tests=$has_ui_changes + elif [[ "$ui_tests_strategy" == "on_renovate_ui_change" ]]; then + if [[ "$has_ui_changes" == "true" && "$is_renovate" == "true" ]]; then + should_run_ui_tests=true + else + should_run_ui_tests=false + fi + else + should_run_ui_tests=false + fi + + echo "ui_tests_needed=$should_run_ui_tests" >> "$GITHUB_OUTPUT" + echo "//// branch=$branch_name has_ui_changes=$has_ui_changes is_renovate=$is_renovate strategy=$ui_tests_strategy ui_tests_needed=$should_run_ui_tests" diff --git a/.github/workflows/test-module.yml b/.github/workflows/test-module.yml index ee3ff6e..9f91741 100644 --- a/.github/workflows/test-module.yml +++ b/.github/workflows/test-module.yml @@ -39,52 +39,15 @@ jobs: echo "node_b=rl1" >> "$GITHUB_OUTPUT" fi - detect_changes: - name: "Detect changes" - runs-on: ubuntu-latest - outputs: - should_run_ui_tests: ${{ steps.changes.outputs.should_run_ui_tests }} - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ github.event.workflow_run.head_sha || github.sha }} - fetch-depth: 2 - - id: changes - run: | - if git diff --name-only HEAD^ HEAD | grep -qE '^ui/|^build-images\.sh$'; then - has_ui_changes=true - else - has_ui_changes=false - fi - - branch_name="${{ github.event.workflow_run.head_branch || github.ref_name }}" - if [[ "$branch_name" == renovate-* || "$branch_name" == renovate/* ]]; then - is_renovate=true - else - is_renovate=false - fi - - ui_tests_strategy="${{ inputs.ui_tests_strategy }}" - if [[ "$ui_tests_strategy" == "never" ]]; then - should_run_ui_tests=false - elif [[ "$ui_tests_strategy" == "on_ui_change" ]]; then - should_run_ui_tests=$has_ui_changes - elif [[ "$ui_tests_strategy" == "on_renovate_ui_change" ]]; then - if [[ "$has_ui_changes" == "true" && "$is_renovate" == "true" ]]; then - should_run_ui_tests=true - else - should_run_ui_tests=false - fi - else - should_run_ui_tests=false - fi - - echo "should_run_ui_tests=$should_run_ui_tests" >> "$GITHUB_OUTPUT" - - echo "//// branch=$branch_name has_ui_changes=$has_ui_changes is_renovate=$is_renovate strategy=$ui_tests_strategy should_run_ui_tests=$should_run_ui_tests" + check_ui_tests_needed: + uses: ./.github/workflows/check-ui-tests-needed.yml + with: + ui_tests_strategy: ${{ inputs.ui_tests_strategy }} + head_sha: ${{ github.event.workflow_run.head_sha || github.sha }} + head_branch: ${{ github.event.workflow_run.head_branch || github.ref_name }} run_tests: - needs: [module, chooser, detect_changes] + needs: [module, chooser, check_ui_tests_needed] strategy: fail-fast: false matrix: @@ -100,7 +63,8 @@ jobs: }} args: ${{ format('ghcr.io/{0}/{1}:{2} -v SCENARIO:{3}', needs.module.outputs.owner, needs.module.outputs.name, needs.module.outputs.tag, matrix.scenario) }} repo_ref: ${{ needs.module.outputs.sha }} - run_ui_tests: ${{ needs.detect_changes.outputs.should_run_ui_tests == 'true' }} + # run_ui_tests: ${{ needs.detect_changes.outputs.ui_tests_needed == 'true' }} //// uncomment + run_ui_tests: true debug_shell: ${{ inputs.debug_shell }} secrets: do_token: ${{ secrets.do_token }} From 47f75fd6d2a5328864770c98bab41a197ffbb73e Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Tue, 14 Apr 2026 17:31:32 +0200 Subject: [PATCH 26/27] hardcode pr for ns8-core --- .github/workflows/test-on-digitalocean-infra.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index 4f53725..573bc58 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -266,7 +266,13 @@ jobs: echo "" >> comment.md done fi - cml comment create --target=pr/252 --repo="https://github.com/${{ github.repository }}" comment.md #### //// TODO: remove hardcoded PR + # //// remove hardcoded PR targets + if [[ "${{ github.repository }}" == */ns8-mail ]]; then + pr_target="pr/252" + elif [[ "${{ github.repository }}" == */ns8-core ]]; then + pr_target="pr/1142" + fi + cml comment create --target="$pr_target" --repo="https://github.com/${{ github.repository }}" comment.md - name: "Debugging with tmate" if: ${{ inputs.debug_shell == true && ! cancelled() && always() }} uses: mxschmitt/action-tmate@v3 From 46a71904450e5186ff1d36fbcf90e9687bab0f75 Mon Sep 17 00:00:00 2001 From: Andrea Leardini Date: Tue, 14 Apr 2026 20:25:57 +0200 Subject: [PATCH 27/27] fix screenshot name --- .github/workflows/test-module.yml | 1 + .github/workflows/test-on-digitalocean-infra.yml | 3 +++ scripts/test-module.sh | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-module.yml b/.github/workflows/test-module.yml index 9f91741..cfe1695 100644 --- a/.github/workflows/test-module.yml +++ b/.github/workflows/test-module.yml @@ -48,6 +48,7 @@ jobs: run_tests: needs: [module, chooser, check_ui_tests_needed] + name: "Run tests" strategy: fail-fast: false matrix: diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index 573bc58..acc902a 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -190,8 +190,10 @@ jobs: id: resolve_script run: | if [ -n "${{ inputs.script }}" ]; then + echo "Using custom script: ${{ inputs.script }} ////" echo "path=${{ github.workspace }}/module/${{ inputs.path }}/${{ inputs.script }}" >> "$GITHUB_OUTPUT" else + echo "Using default script: ns8-github-actions/scripts/test-module.sh ////" echo "path=${{ github.workspace }}/ns8-github-actions/scripts/test-module.sh" >> "$GITHUB_OUTPUT" fi - name: "Run tests" @@ -260,6 +262,7 @@ jobs: else for f in "${screenshots[@]}"; do name=$(basename "${f%.*}") + name="${name//_/ }" echo "**${name^}**" >> comment.md echo "" >> comment.md echo "![$(basename "$f")]($f)" >> comment.md diff --git a/scripts/test-module.sh b/scripts/test-module.sh index f53e513..49be231 100644 --- a/scripts/test-module.sh +++ b/scripts/test-module.sh @@ -36,10 +36,12 @@ if [ "${RUN_UI_TESTS}" = "true" ]; then container_image="mcr.microsoft.com/playwright:v1.51.0-noble" container_shell="bash" pythonreq="/srv/ns8-github-actions/tests/pythonreq-ui.txt" + cache_volume="rftest-cache-ui" else container_image="docker.io/python:3.11-alpine" container_shell="ash" pythonreq="/srv/ns8-github-actions/tests/pythonreq.txt" + cache_volume="rftest-cache" fi # Run the test suite inside a container. @@ -51,7 +53,7 @@ fi podman run -i \ --volume=.:/srv/source:z \ --volume=${script_dir}/tests:/srv/ns8-github-actions/tests:z \ - --volume=rftest-cache:${venvroot}:z \ + --volume=${cache_volume}:${venvroot}:z \ --replace --name=rftest \ --env=ssh_key \ --env=venvroot \