diff --git a/.github/workflows/check-ui-tests-needed.yml b/.github/workflows/check-ui-tests-needed.yml new file mode 100644 index 0000000..7d6e404 --- /dev/null +++ b/.github/workflows/check-ui-tests-needed.yml @@ -0,0 +1,76 @@ +name: Check UI tests needed + +on: + workflow_call: + inputs: + ui_tests_strategy: + description: "When to run UI tests: on_ui_change, on_renovate_ui_change, never" + required: false + type: string + default: on_renovate_ui_change + head_sha: + description: "The commit SHA to checkout" + required: true + type: string + head_branch: + description: "The branch name to check for renovate" + required: true + type: string + ui_dir: + description: "The UI directory to watch for changes" + required: false + type: string + default: ui + build_script: + description: "The build script file to watch for changes (changes to this file may affect the UI)" + required: false + type: string + default: build-images.sh + outputs: + ui_tests_needed: + description: "Whether UI tests should run based on strategy and detected changes" + value: ${{ jobs.detect_changes.outputs.ui_tests_needed }} + +jobs: + detect_changes: + name: "Detect changes" + runs-on: ubuntu-latest + outputs: + ui_tests_needed: ${{ steps.changes.outputs.ui_tests_needed }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.head_sha }} + fetch-depth: 2 + - id: changes + run: | + if git diff --name-only HEAD^ HEAD | grep -qE "^${{ inputs.ui_dir }}/|^${{ inputs.build_script }}$"; then + has_ui_changes=true + else + has_ui_changes=false + fi + + branch_name="${{ inputs.head_branch }}" + if [[ "$branch_name" == renovate-* || "$branch_name" == renovate/* ]]; then + is_renovate=true + else + is_renovate=false + fi + + ui_tests_strategy="${{ inputs.ui_tests_strategy }}" + if [[ "$ui_tests_strategy" == "never" ]]; then + should_run_ui_tests=false + elif [[ "$ui_tests_strategy" == "on_ui_change" ]]; then + should_run_ui_tests=$has_ui_changes + elif [[ "$ui_tests_strategy" == "on_renovate_ui_change" ]]; then + if [[ "$has_ui_changes" == "true" && "$is_renovate" == "true" ]]; then + should_run_ui_tests=true + else + should_run_ui_tests=false + fi + else + should_run_ui_tests=false + fi + + echo "ui_tests_needed=$should_run_ui_tests" >> "$GITHUB_OUTPUT" + echo "//// branch=$branch_name has_ui_changes=$has_ui_changes is_renovate=$is_renovate strategy=$ui_tests_strategy ui_tests_needed=$should_run_ui_tests" diff --git a/.github/workflows/module-info.yml b/.github/workflows/module-info.yml index 99358a2..eb97f81 100644 --- a/.github/workflows/module-info.yml +++ b/.github/workflows/module-info.yml @@ -31,6 +31,7 @@ on: jobs: module: + name: "Get module info" runs-on: ubuntu-latest outputs: owner: ${{ steps.owner.outputs.name }} diff --git a/.github/workflows/test-module.yml b/.github/workflows/test-module.yml new file mode 100644 index 0000000..cfe1695 --- /dev/null +++ b/.github/workflows/test-module.yml @@ -0,0 +1,71 @@ +name: Test module + +on: + workflow_call: + inputs: + ui_tests_strategy: + description: "When to run UI tests: on_ui_change, on_renovate_ui_change, never" + required: false + type: string + default: on_renovate_ui_change + debug_shell: + description: "Debug shell" + required: false + type: boolean + default: false + secrets: + do_token: + required: true + +jobs: + module: + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == '' }} + uses: ./.github/workflows/module-info.yml + + chooser: + name: "Choose nodes" + runs-on: ubuntu-latest + outputs: + node_a: ${{ steps.pick.outputs.node_a }} + node_b: ${{ steps.pick.outputs.node_b }} + steps: + - id: pick + run: | + if (( $GITHUB_RUN_NUMBER % 2 )); then + echo "node_a=rl1" >> "$GITHUB_OUTPUT" + echo "node_b=dn1" >> "$GITHUB_OUTPUT" + else + echo "node_a=dn1" >> "$GITHUB_OUTPUT" + echo "node_b=rl1" >> "$GITHUB_OUTPUT" + fi + + check_ui_tests_needed: + uses: ./.github/workflows/check-ui-tests-needed.yml + with: + ui_tests_strategy: ${{ inputs.ui_tests_strategy }} + head_sha: ${{ github.event.workflow_run.head_sha || github.sha }} + head_branch: ${{ github.event.workflow_run.head_branch || github.ref_name }} + + run_tests: + needs: [module, chooser, check_ui_tests_needed] + name: "Run tests" + strategy: + fail-fast: false + matrix: + scenario: [install, update] + uses: ./.github/workflows/test-on-digitalocean-infra.yml + with: + coremodules: ${{ matrix.scenario == 'install' && format('ghcr.io/{0}/{1}:{2}', needs.module.outputs.owner, needs.module.outputs.name, needs.module.outputs.tag) || '' }} + leader_nodes: >- + ${{ + matrix.scenario == 'install' + && needs.chooser.outputs.node_a + || needs.chooser.outputs.node_b + }} + args: ${{ format('ghcr.io/{0}/{1}:{2} -v SCENARIO:{3}', needs.module.outputs.owner, needs.module.outputs.name, needs.module.outputs.tag, matrix.scenario) }} + repo_ref: ${{ needs.module.outputs.sha }} + # run_ui_tests: ${{ needs.detect_changes.outputs.ui_tests_needed == 'true' }} //// uncomment + run_ui_tests: true + debug_shell: ${{ inputs.debug_shell }} + secrets: + do_token: ${{ secrets.do_token }} diff --git a/.github/workflows/test-on-digitalocean-infra.yml b/.github/workflows/test-on-digitalocean-infra.yml index af5d0d5..acc902a 100644 --- a/.github/workflows/test-on-digitalocean-infra.yml +++ b/.github/workflows/test-on-digitalocean-infra.yml @@ -5,13 +5,13 @@ on: inputs: script: required: false - description: "Name of the script to execute" + description: "Name of the script to execute. If empty, the default script from ns8-github-actions is used." type: string - default: test-module.sh + default: "" path: required: false type: string - description: "Name of the script to execute, default `test-module.sh`" + description: "Subdirectory of the module repository to use as working directory when running the test script" args: required: false type: string @@ -55,6 +55,11 @@ on: type: string default: "ci.nethserver.net" description: "DigitalOcean domain where to create the DNS records" + run_ui_tests: + required: false + type: boolean + default: false + description: "If true, run UI tests" secrets: do_token: required: true @@ -117,6 +122,15 @@ jobs: with: ref: ${{ inputs.repo_ref }} path: module + - name: "Get ns8-github-actions ref" + id: ns8_ga_ref + run: echo "ref=$(echo '${{ github.workflow_ref }}' | cut -d'@' -f2)" >> $GITHUB_OUTPUT + - name: "Checkout ns8-github-actions" + uses: actions/checkout@v6 + with: + repository: NethServer/ns8-github-actions + ref: ${{ steps.ns8_ga_ref.outputs.ref }} + path: ns8-github-actions - name: "Checkout the infrastructure repository" uses: actions/checkout@v6 with: @@ -172,14 +186,26 @@ jobs: set -e -x create-cluster ${{ matrix.node }}.leader.${{ matrix.node }}-${{ needs.info.outputs.short_sha }}.${{ env.TF_VAR_domain }}:55820 10.5.4.0/24 Nethesis,1234 EOF + - name: "Resolve test script" + id: resolve_script + run: | + if [ -n "${{ inputs.script }}" ]; then + echo "Using custom script: ${{ inputs.script }} ////" + echo "path=${{ github.workspace }}/module/${{ inputs.path }}/${{ inputs.script }}" >> "$GITHUB_OUTPUT" + else + echo "Using default script: ns8-github-actions/scripts/test-module.sh ////" + echo "path=${{ github.workspace }}/ns8-github-actions/scripts/test-module.sh" >> "$GITHUB_OUTPUT" + fi - name: "Run tests" id: run_tests run: | - ./${{ inputs.script }} ${{ matrix.node }}.leader.${{ matrix.node }}-${{ needs.info.outputs.short_sha }}.${{ env.TF_VAR_domain }} ${{ inputs.args }} + echo "Running test script: ${{ steps.resolve_script.outputs.path }} ////" + bash "${{ steps.resolve_script.outputs.path }}" ${{ matrix.node }}.leader.${{ matrix.node }}-${{ needs.info.outputs.short_sha }}.${{ env.TF_VAR_domain }} ${{ inputs.args }} working-directory: ${{ github.workspace }}/module/${{ inputs.path }} env: SSH_KEYFILE: ${{ github.workspace }}/key COREMODULES: ${{ inputs.coremodules }} + RUN_UI_TESTS: ${{ inputs.run_ui_tests }} - name: "Save tests results" if: always() uses: actions/upload-artifact@v6 @@ -219,6 +245,37 @@ jobs: -f "target_url=$status_page" \ -f "description=Test on DigitalOcean (${{ matrix.node }})" \ -f "context=continuous-integration/robotframework/${{ matrix.node }})" + - name: "Setup CML" + if: inputs.run_ui_tests == true + run: npm install --location=global @dvcorg/cml + - name: "Post screenshots to PR" + if: inputs.run_ui_tests == true + env: + REPO_TOKEN: ${{ github.token }} + MODULE_PATH: ${{ inputs.path }} + run: | + output_dir="${{ github.workspace }}/module${MODULE_PATH:+/$MODULE_PATH}/tests/outputs" + echo "### UI Test Screenshots \`${{ matrix.node }}\`" >> comment.md + mapfile -d '' screenshots < <(find "$output_dir" -type f \( -iname '*.png' -o -iname '*.jpg' -o -iname '*.jpeg' -o -iname '*.gif' -o -iname '*.webp' \) -print0 2>/dev/null | sort -z) + if [ ${#screenshots[@]} -eq 0 ]; then + echo "No screenshots found in test outputs." >> comment.md + else + for f in "${screenshots[@]}"; do + name=$(basename "${f%.*}") + name="${name//_/ }" + echo "**${name^}**" >> comment.md + echo "" >> comment.md + echo "![$(basename "$f")]($f)" >> comment.md + echo "" >> comment.md + done + fi + # //// remove hardcoded PR targets + if [[ "${{ github.repository }}" == */ns8-mail ]]; then + pr_target="pr/252" + elif [[ "${{ github.repository }}" == */ns8-core ]]; then + pr_target="pr/1142" + fi + cml comment create --target="$pr_target" --repo="https://github.com/${{ github.repository }}" comment.md - name: "Debugging with tmate" if: ${{ inputs.debug_shell == true && ! cancelled() && always() }} uses: mxschmitt/action-tmate@v3 diff --git a/scripts/test-module.sh b/scripts/test-module.sh new file mode 100644 index 0000000..49be231 --- /dev/null +++ b/scripts/test-module.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +set -e -a + +# //// +_script_start=$(date +%s) + +# Path to the SSH private key used to connect to the NS8 leader node +SSH_KEYFILE=${SSH_KEYFILE:-$HOME/.ssh/id_rsa} + +# Mandatory positional arguments: the leader node hostname and the module image URL +LEADER_NODE="${1:?missing LEADER_NODE argument}" +IMAGE_URL="${2:?missing IMAGE_URL argument}" +shift 2 + +# Read SSH key contents so they can be injected into the container via an env var +ssh_key="$(<$SSH_KEYFILE)" + +# The venv is stored in a named volume (rftest-cache) to cache pip/rfbrowser installs across runs +venvroot=/usr/local/venv + +# Resolve the directory of this script to mount the shared test requirements inside the container +script_dir="$(cd "$(dirname "$0")" && pwd)" + +echo "Test! RUN_UI_TESTS=${RUN_UI_TESTS} ////" + +# Select the container image and Python requirements file based on whether UI tests are enabled. +# UI tests require the Playwright image (Debian-based, includes browser binaries). +# Non-UI tests use a lightweight Alpine Python image. +if [ "${RUN_UI_TESTS}" = "true" ]; then + container_image="mcr.microsoft.com/playwright:v1.51.0-noble" + container_shell="bash" + pythonreq="/srv/ns8-github-actions/tests/pythonreq-ui.txt" + cache_volume="rftest-cache-ui" +else + container_image="docker.io/python:3.11-alpine" + container_shell="ash" + pythonreq="/srv/ns8-github-actions/tests/pythonreq.txt" + cache_volume="rftest-cache" +fi + +# Run the test suite inside a container. +# Mounts: +# . → /srv/source (module source tree, including the tests/ directory) +# scripts/tests → /srv/ns8-github-actions/tests (shared Robot Framework helpers and requirements) +# rftest-cache → ${venvroot} (named volume to persist the Python venv across runs) +# Any extra arguments after IMAGE_URL are forwarded to the robot command inside the container. +podman run -i \ + --volume=.:/srv/source:z \ + --volume=${script_dir}/tests:/srv/ns8-github-actions/tests:z \ + --volume=${cache_volume}:${venvroot}:z \ + --replace --name=rftest \ + --env=ssh_key \ + --env=venvroot \ + --env=LEADER_NODE \ + --env=IMAGE_URL \ + --env=RUN_UI_TESTS \ + --env=pythonreq \ + --env=_script_start \ + "${container_image}" \ + ${container_shell} -l -s -- "${@}" <<'EOF' +set -e + +# Write the SSH private key to a temp file for use by Robot Framework tests +echo "$ssh_key" > /tmp/idssh + +# Install the Python venv and Robot Framework dependencies if not already cached. +# Cache is invalidated when the MD5 checksum of the requirements file changes, ensuring +# that dependency updates are always picked up even on reused (self-hosted) runners. +pythonreq_checksum_file="${venvroot}/.pythonreq.md5" +pythonreq_current_checksum=$(md5sum "${pythonreq}" | cut -d' ' -f1) +pythonreq_cached_checksum=$(cat "${pythonreq_checksum_file}" 2>/dev/null || true) + +if [ ! -x "${venvroot}/bin/robot" ] || [ "${pythonreq_current_checksum}" != "${pythonreq_cached_checksum}" ] ; then + if command -v apt-get > /dev/null 2>&1; then + # mcr.microsoft.com/playwright:*-noble has npm pre-installed but no Python + apt-get update -q + apt-get install -y -q python3 python3-venv + python3 -mvenv "${venvroot}" + else + # Alpine image already has Python; --upgrade refreshes pip/setuptools in-place + python3 -mvenv "${venvroot}" --upgrade + fi + ${venvroot}/bin/pip3 install -q -r "${pythonreq}" + # Save the checksum so future runs can detect requirement changes + echo "${pythonreq_current_checksum}" > "${pythonreq_checksum_file}" + # Invalidate the rfbrowser sentinel so it is re-initialized with the new packages + rm -f "${venvroot}/.rfbrowser_initialized" +fi + +# Initialize the Playwright browser binaries for rfbrowser (UI tests only). +# A sentinel file prevents re-running this expensive step on cache hits. +if [ "${RUN_UI_TESTS}" = "true" ] && [ ! -f "${venvroot}/.rfbrowser_initialized" ] ; then + ${venvroot}/bin/rfbrowser init + touch "${venvroot}/.rfbrowser_initialized" +fi + +cd /srv/source +mkdir -vp tests/outputs/ + +# Exclude UI tests if RUN_UI_TESTS is not set to "true" +if [ "${RUN_UI_TESTS}" = "true" ]; then + ui_tag_filter="" +else + ui_tag_filter="--exclude ui" +fi + +echo "DEBUG: $(( $(date +%s) - _script_start ))s elapsed from script start to robot launch ////" + +exec ${venvroot}/bin/robot \ + -v NODE_ADDR:${LEADER_NODE} \ + -v IMAGE_URL:${IMAGE_URL} \ + -v SSH_KEYFILE:/tmp/idssh \ + -v RUN_UI_TESTS:${RUN_UI_TESTS} \ + --name test-ns8-module \ + --skiponfailure unstable \ + ${ui_tag_filter} \ + -d tests/outputs "${@}" tests/ +EOF diff --git a/scripts/tests/pythonreq-ui.txt b/scripts/tests/pythonreq-ui.txt new file mode 100644 index 0000000..4cd8c1f --- /dev/null +++ b/scripts/tests/pythonreq-ui.txt @@ -0,0 +1,2 @@ +-r pythonreq.txt +robotframework-browser diff --git a/scripts/tests/pythonreq.txt b/scripts/tests/pythonreq.txt new file mode 100644 index 0000000..af1d9bf --- /dev/null +++ b/scripts/tests/pythonreq.txt @@ -0,0 +1,2 @@ +robotframework +robotframework-sshlibrary