feat: centralize module testing

.github/workflows/check-ui-tests-needed.yml

@@ -0,0 +1,76 @@
+name: Check UI tests needed
+
+on:
+  workflow_call:
+    inputs:
+      ui_tests_strategy:
+        description: "When to run UI tests: on_ui_change, on_renovate_ui_change, never"
+        required: false
+        type: string
+        default: on_renovate_ui_change
+      head_sha:
+        description: "The commit SHA to checkout"
+        required: true
+        type: string
+      head_branch:
+        description: "The branch name to check for renovate"
+        required: true
+        type: string
+      ui_dir:
+        description: "The UI directory to watch for changes"
+        required: false
+        type: string
+        default: ui
+      build_script:
+        description: "The build script file to watch for changes (changes to this file may affect the UI)"
+        required: false
+        type: string
+        default: build-images.sh
+    outputs:
+      ui_tests_needed:
+        description: "Whether UI tests should run based on strategy and detected changes"
+        value: ${{ jobs.detect_changes.outputs.ui_tests_needed }}
+
+jobs:
+  detect_changes:
+    name: "Detect changes"
+    runs-on: ubuntu-latest
+    outputs:
+      ui_tests_needed: ${{ steps.changes.outputs.ui_tests_needed }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.head_sha }}
+          fetch-depth: 2
+      - id: changes
+        run: |
+          if git diff --name-only HEAD^ HEAD | grep -qE "^${{ inputs.ui_dir }}/|^${{ inputs.build_script }}$"; then
+            has_ui_changes=true
+          else
+            has_ui_changes=false
+          fi
+
+          branch_name="${{ inputs.head_branch }}"
+          if [[ "$branch_name" == renovate-* || "$branch_name" == renovate/* ]]; then
+            is_renovate=true
+          else
+            is_renovate=false
+          fi
+
+          ui_tests_strategy="${{ inputs.ui_tests_strategy }}"
+          if [[ "$ui_tests_strategy" == "never" ]]; then
+            should_run_ui_tests=false
+          elif [[ "$ui_tests_strategy" == "on_ui_change" ]]; then
+            should_run_ui_tests=$has_ui_changes
+          elif [[ "$ui_tests_strategy" == "on_renovate_ui_change" ]]; then
+            if [[ "$has_ui_changes" == "true" && "$is_renovate" == "true" ]]; then
+              should_run_ui_tests=true
+            else
+              should_run_ui_tests=false
+            fi
+          else
+            should_run_ui_tests=false
+          fi
+
+          echo "ui_tests_needed=$should_run_ui_tests" >> "$GITHUB_OUTPUT"
+          echo "//// branch=$branch_name has_ui_changes=$has_ui_changes is_renovate=$is_renovate strategy=$ui_tests_strategy ui_tests_needed=$should_run_ui_tests"

.github/workflows/module-info.yml

@@ -31,6 +31,7 @@ on:
 
 jobs:
   module:
+    name: "Get module info"
     runs-on: ubuntu-latest
     outputs:
       owner: ${{ steps.owner.outputs.name }}

.github/workflows/test-module.yml

@@ -0,0 +1,71 @@
+name: Test module
+
+on:
+  workflow_call:
+    inputs:
+      ui_tests_strategy:
+        description: "When to run UI tests: on_ui_change, on_renovate_ui_change, never"
+        required: false
+        type: string
+        default: on_renovate_ui_change
+      debug_shell:
+        description: "Debug shell"
+        required: false
+        type: boolean
+        default: false
+    secrets:
+      do_token:
+        required: true
+
+jobs:
+  module:
+    if: ${{ github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == '' }}
+    uses: ./.github/workflows/module-info.yml
+
+  chooser:
+    name: "Choose nodes"
+    runs-on: ubuntu-latest
+    outputs:
+      node_a: ${{ steps.pick.outputs.node_a }}
+      node_b: ${{ steps.pick.outputs.node_b }}
+    steps:
+      - id: pick
+        run: |
+          if (( $GITHUB_RUN_NUMBER % 2 )); then
+            echo "node_a=rl1" >> "$GITHUB_OUTPUT"
+            echo "node_b=dn1" >> "$GITHUB_OUTPUT"
+          else
+            echo "node_a=dn1" >> "$GITHUB_OUTPUT"
+            echo "node_b=rl1" >> "$GITHUB_OUTPUT"
+          fi
+
+  check_ui_tests_needed:
+    uses: ./.github/workflows/check-ui-tests-needed.yml
+    with:
+      ui_tests_strategy: ${{ inputs.ui_tests_strategy }}
+      head_sha: ${{ github.event.workflow_run.head_sha || github.sha }}
+      head_branch: ${{ github.event.workflow_run.head_branch || github.ref_name }}
+
+  run_tests:
+    needs: [module, chooser, check_ui_tests_needed]
+    name: "Run tests"
+    strategy:
+      fail-fast: false
+      matrix:
+        scenario: [install, update]
+    uses: ./.github/workflows/test-on-digitalocean-infra.yml
+    with:
+      coremodules: ${{ matrix.scenario == 'install' && format('ghcr.io/{0}/{1}:{2}', needs.module.outputs.owner, needs.module.outputs.name, needs.module.outputs.tag) || '' }}
+      leader_nodes: >-
+        ${{
+          matrix.scenario == 'install'
+          && needs.chooser.outputs.node_a
+          || needs.chooser.outputs.node_b
+        }}
+      args: ${{ format('ghcr.io/{0}/{1}:{2} -v SCENARIO:{3}', needs.module.outputs.owner, needs.module.outputs.name, needs.module.outputs.tag, matrix.scenario) }}
+      repo_ref: ${{ needs.module.outputs.sha }}
+      # run_ui_tests: ${{ needs.detect_changes.outputs.ui_tests_needed == 'true' }} //// uncomment
+      run_ui_tests: true
+      debug_shell: ${{ inputs.debug_shell }}
+    secrets:
+      do_token: ${{ secrets.do_token }}

.github/workflows/test-on-digitalocean-infra.yml

@@ -5,13 +5,13 @@ on:
     inputs:
       script:
         required: false
-        description: "Name of the script to execute"
+        description: "Name of the script to execute. If empty, the default script from ns8-github-actions is used."
         type: string
-        default: test-module.sh
+        default: ""
       path:
         required: false
         type: string
-        description: "Name of the script to execute, default `test-module.sh`"
+        description: "Subdirectory of the module repository to use as working directory when running the test script"
       args:
         required: false
         type: string
@@ -55,6 +55,11 @@ on:
         type: string
         default: "ci.nethserver.net"
         description: "DigitalOcean domain where to create the DNS records"
+      run_ui_tests:
+        required: false
+        type: boolean
+        default: false
+        description: "If true, run UI tests"
     secrets:
       do_token:
         required: true
@@ -117,6 +122,15 @@ jobs:
         with:
           ref: ${{ inputs.repo_ref }}
           path: module
+      - name: "Get ns8-github-actions ref"
+        id: ns8_ga_ref
+        run: echo "ref=$(echo '${{ github.workflow_ref }}' | cut -d'@' -f2)" >> $GITHUB_OUTPUT
+      - name: "Checkout ns8-github-actions"
+        uses: actions/checkout@v6
+        with:
+          repository: NethServer/ns8-github-actions
+          ref: ${{ steps.ns8_ga_ref.outputs.ref }}
+          path: ns8-github-actions
       - name: "Checkout the infrastructure repository"
         uses: actions/checkout@v6
         with:
@@ -172,14 +186,26 @@ jobs:
             set -e -x
             create-cluster ${{ matrix.node }}.leader.${{ matrix.node }}-${{ needs.info.outputs.short_sha }}.${{ env.TF_VAR_domain }}:55820 10.5.4.0/24 Nethesis,1234
           EOF
+      - name: "Resolve test script"
+        id: resolve_script
+        run: |
+          if [ -n "${{ inputs.script }}" ]; then
+            echo "Using custom script: ${{ inputs.script }} ////"
+            echo "path=${{ github.workspace }}/module/${{ inputs.path }}/${{ inputs.script }}" >> "$GITHUB_OUTPUT"
+          else
+            echo "Using default script: ns8-github-actions/scripts/test-module.sh ////"
+            echo "path=${{ github.workspace }}/ns8-github-actions/scripts/test-module.sh" >> "$GITHUB_OUTPUT"
+          fi
       - name: "Run tests"
         id: run_tests
         run: |
-          ./${{ inputs.script }} ${{ matrix.node }}.leader.${{ matrix.node }}-${{ needs.info.outputs.short_sha }}.${{ env.TF_VAR_domain }} ${{ inputs.args }}
+          echo "Running test script: ${{ steps.resolve_script.outputs.path }} ////"
+          bash "${{ steps.resolve_script.outputs.path }}" ${{ matrix.node }}.leader.${{ matrix.node }}-${{ needs.info.outputs.short_sha }}.${{ env.TF_VAR_domain }} ${{ inputs.args }}
         working-directory: ${{ github.workspace }}/module/${{ inputs.path }}
         env:
           SSH_KEYFILE: ${{ github.workspace }}/key
           COREMODULES: ${{ inputs.coremodules }}
+          RUN_UI_TESTS: ${{ inputs.run_ui_tests }}
       - name: "Save tests results"
         if: always()
         uses: actions/upload-artifact@v6
@@ -219,6 +245,37 @@ jobs:
             -f "target_url=$status_page" \
             -f "description=Test on DigitalOcean (${{ matrix.node }})" \
             -f "context=continuous-integration/robotframework/${{ matrix.node }})"
+      - name: "Setup CML"
+        if: inputs.run_ui_tests == true
+        run: npm install --location=global @dvcorg/cml
+      - name: "Post screenshots to PR"
+        if: inputs.run_ui_tests == true
+        env:
+          REPO_TOKEN: ${{ github.token }}
+          MODULE_PATH: ${{ inputs.path }}
+        run: |
+          output_dir="${{ github.workspace }}/module${MODULE_PATH:+/$MODULE_PATH}/tests/outputs"
+          echo "### UI Test Screenshots \`${{ matrix.node }}\`" >> comment.md
+          mapfile -d '' screenshots < <(find "$output_dir" -type f \( -iname '*.png' -o -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.gif' -o -iname '*.webp' \) -print0 2>/dev/null | sort -z)
+          if [ ${#screenshots[@]} -eq 0 ]; then
+            echo "No screenshots found in test outputs." >> comment.md
+          else
+            for f in "${screenshots[@]}"; do
+              name=$(basename "${f%.*}")
+              name="${name//_/ }"
+              echo "**${name^}**" >> comment.md
+              echo "" >> comment.md
+              echo "![$(basename "$f")]($f)" >> comment.md
+              echo "" >> comment.md
+            done
+          fi
+          # //// remove hardcoded PR targets
+          if [[ "${{ github.repository }}" == */ns8-mail ]]; then
+            pr_target="pr/252"
+          elif [[ "${{ github.repository }}" == */ns8-core ]]; then
+            pr_target="pr/1142"
+          fi
+          cml comment create --target="$pr_target" --repo="https://github.com/${{ github.repository }}" comment.md
       - name: "Debugging with tmate"
         if: ${{ inputs.debug_shell == true && ! cancelled() && always() }}
         uses: mxschmitt/action-tmate@v3

scripts/test-module.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+
+#
+# Copyright (C) 2026 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+set -e -a
+
+# ////
+_script_start=$(date +%s)
+
+# Path to the SSH private key used to connect to the NS8 leader node
+SSH_KEYFILE=${SSH_KEYFILE:-$HOME/.ssh/id_rsa}
+
+# Mandatory positional arguments: the leader node hostname and the module image URL
+LEADER_NODE="${1:?missing LEADER_NODE argument}"
+IMAGE_URL="${2:?missing IMAGE_URL argument}"
+shift 2
+
+# Read SSH key contents so they can be injected into the container via an env var
+ssh_key="$(<$SSH_KEYFILE)"
+
+# The venv is stored in a named volume (rftest-cache) to cache pip/rfbrowser installs across runs
+venvroot=/usr/local/venv
+
+# Resolve the directory of this script to mount the shared test requirements inside the container
+script_dir="$(cd "$(dirname "$0")" && pwd)"
+
+echo "Test! RUN_UI_TESTS=${RUN_UI_TESTS} ////"
+
+# Select the container image and Python requirements file based on whether UI tests are enabled.
+# UI tests require the Playwright image (Debian-based, includes browser binaries).
+# Non-UI tests use a lightweight Alpine Python image.
+if [ "${RUN_UI_TESTS}" = "true" ]; then
+    container_image="mcr.microsoft.com/playwright:v1.51.0-noble"
+    container_shell="bash"
+    pythonreq="/srv/ns8-github-actions/tests/pythonreq-ui.txt"
+    cache_volume="rftest-cache-ui"
+else
+    container_image="docker.io/python:3.11-alpine"
+    container_shell="ash"
+    pythonreq="/srv/ns8-github-actions/tests/pythonreq.txt"
+    cache_volume="rftest-cache"
+fi
+
+# Run the test suite inside a container.
+# Mounts:
+#   .              → /srv/source          (module source tree, including the tests/ directory)
+#   scripts/tests  → /srv/ns8-github-actions/tests  (shared Robot Framework helpers and requirements)
+#   rftest-cache   → ${venvroot}          (named volume to persist the Python venv across runs)
+# Any extra arguments after IMAGE_URL are forwarded to the robot command inside the container.
+podman run -i \
+    --volume=.:/srv/source:z \
+    --volume=${script_dir}/tests:/srv/ns8-github-actions/tests:z \
+    --volume=${cache_volume}:${venvroot}:z \
+    --replace --name=rftest \
+    --env=ssh_key \
+    --env=venvroot \
+    --env=LEADER_NODE \
+    --env=IMAGE_URL \
+    --env=RUN_UI_TESTS \
+    --env=pythonreq \
+    --env=_script_start \
+    "${container_image}" \
+    ${container_shell} -l -s -- "${@}" <<'EOF'
+set -e
+
+# Write the SSH private key to a temp file for use by Robot Framework tests
+echo "$ssh_key" > /tmp/idssh
+
+# Install the Python venv and Robot Framework dependencies if not already cached.
+# Cache is invalidated when the MD5 checksum of the requirements file changes, ensuring
+# that dependency updates are always picked up even on reused (self-hosted) runners.
+pythonreq_checksum_file="${venvroot}/.pythonreq.md5"
+pythonreq_current_checksum=$(md5sum "${pythonreq}" | cut -d' ' -f1)
+pythonreq_cached_checksum=$(cat "${pythonreq_checksum_file}" 2>/dev/null || true)
+
+if [ ! -x "${venvroot}/bin/robot" ] || [ "${pythonreq_current_checksum}" != "${pythonreq_cached_checksum}" ] ; then
+    if command -v apt-get > /dev/null 2>&1; then
+        # mcr.microsoft.com/playwright:*-noble has npm pre-installed but no Python
+        apt-get update -q
+        apt-get install -y -q python3 python3-venv
+        python3 -mvenv "${venvroot}"
+    else
+        # Alpine image already has Python; --upgrade refreshes pip/setuptools in-place
+        python3 -mvenv "${venvroot}" --upgrade
+    fi
+    ${venvroot}/bin/pip3 install -q -r "${pythonreq}"
+    # Save the checksum so future runs can detect requirement changes
+    echo "${pythonreq_current_checksum}" > "${pythonreq_checksum_file}"
+    # Invalidate the rfbrowser sentinel so it is re-initialized with the new packages
+    rm -f "${venvroot}/.rfbrowser_initialized"
+fi
+
+# Initialize the Playwright browser binaries for rfbrowser (UI tests only).
+# A sentinel file prevents re-running this expensive step on cache hits.
+if [ "${RUN_UI_TESTS}" = "true" ] && [ ! -f "${venvroot}/.rfbrowser_initialized" ] ; then
+    ${venvroot}/bin/rfbrowser init
+    touch "${venvroot}/.rfbrowser_initialized"
+fi
+
+cd /srv/source
+mkdir -vp tests/outputs/
+
+# Exclude UI tests if RUN_UI_TESTS is not set to "true"
+if [ "${RUN_UI_TESTS}" = "true" ]; then
+    ui_tag_filter=""
+else
+    ui_tag_filter="--exclude ui"
+fi
+
+echo "DEBUG: $(( $(date +%s) - _script_start ))s elapsed from script start to robot launch ////"
+
+exec ${venvroot}/bin/robot \
+    -v NODE_ADDR:${LEADER_NODE} \
+    -v IMAGE_URL:${IMAGE_URL} \
+    -v SSH_KEYFILE:/tmp/idssh \
+    -v RUN_UI_TESTS:${RUN_UI_TESTS} \
+    --name test-ns8-module \
+    --skiponfailure unstable \
+    ${ui_tag_filter} \
+    -d tests/outputs "${@}" tests/
+EOF

scripts/tests/pythonreq-ui.txt

@@ -0,0 +1,2 @@
+-r pythonreq.txt
+robotframework-browser

scripts/tests/pythonreq.txt

@@ -0,0 +1,2 @@
+robotframework
+robotframework-sshlibrary