diff --git a/.github/testflinger-assets/env_setup.sh b/.github/testflinger-assets/env_setup.sh new file mode 100644 index 00000000..6174552f --- /dev/null +++ b/.github/testflinger-assets/env_setup.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# Set to default values. Will be set in snap_integration_tests.yml +export SNAP_TEST_SOURCE="${SNAP_TEST_SOURCE:-local}" +export SNAP_CHANNEL="${SNAP_CHANNEL:-${SNAP_TEST_SOURCE}}" \ No newline at end of file diff --git a/.github/testflinger-assets/test_snap_device_script.sh b/.github/testflinger-assets/test_snap_device_script.sh index 3834d63d..12728c53 100755 --- a/.github/testflinger-assets/test_snap_device_script.sh +++ b/.github/testflinger-assets/test_snap_device_script.sh @@ -16,45 +16,23 @@ while sudo snap debug state /var/lib/snapd/state.json | grep -qE 'Doing|Undoing| done echo " --- Disabling auto-refresh for 24 hours" sudo snap refresh --hold=24h -echo " --- Extracting snap package" -if [[ ! -f ./fpgad-snap-package.tar.gz ]]; then - echo "ERROR: Snap package tarball not found" - exit 1 -fi -tar -xzvf fpgad-snap-package.tar.gz -echo " --- Listing extracted snap and component files" -echo "Snap files:" -find . -maxdepth 1 -name "*.snap" -type f -echo "" -echo "Component files:" -find . -maxdepth 1 -name "*.comp" -type f -echo "" -echo " --- Installing fpgad snap" -SNAP_FILE=$(find . -maxdepth 1 -name "*.snap" -type f | head -n 1) -if [[ -z "$SNAP_FILE" || ! -f "$SNAP_FILE" ]]; then - echo "ERROR: Snap file not found in tarball" - exit 1 -fi +echo " --- Installing fpgad" while sudo snap debug state /var/lib/snapd/state.json | grep -qE 'Doing|Undoing|Waiting'; do echo " --- snapd internal tasks still running... waiting..." sleep 10 done -sudo snap install "$SNAP_FILE" --dangerous -echo " --- Installing snap components" -COMP_FILES=$(find . -maxdepth 1 -name "*.comp" -type f) -if [[ -z "$COMP_FILES" ]]; then - echo "ERROR: No component files found in tarball - build may have failed" - exit 1 + +if [[ "${SNAP_TEST_SOURCE}" == "local" ]]; then + if [[ ! -f ./fpgad.snap ]]; then + echo "ERROR: SNAP_TEST_SOURCE=local but ./fpgad.snap is not present" + exit 1 + fi + echo " --- SNAP_TEST_SOURCE=local, installing ./fpgad.snap --dangerous" + sudo snap install ./fpgad.snap --dangerous +else + echo " --- SNAP_TEST_SOURCE=${SNAP_TEST_SOURCE}, installing from store channel: ${SNAP_TEST_SOURCE}" + sudo snap install fpgad --channel="${SNAP_TEST_SOURCE}" fi -for COMP_FILE in $COMP_FILES; do - echo " --- Installing component: $COMP_FILE" - while sudo snap debug state /var/lib/snapd/state.json | grep -qE 'Doing|Undoing|Waiting'; do - echo " --- snapd internal tasks still running... waiting..." - sleep 10 - done - sudo snap install --dangerous "$COMP_FILE" - echo " --- Component installed successfully: $COMP_FILE" -done echo " --- Installing provider snap(s)" echo "INFO: Done preparing device" diff --git a/.github/testflinger-assets/testflinger_job.yaml b/.github/testflinger-assets/testflinger_job.yaml index e52b250b..933ddc60 100644 --- a/.github/testflinger-assets/testflinger_job.yaml +++ b/.github/testflinger-assets/testflinger_job.yaml @@ -33,6 +33,8 @@ test_data: local: ./fpgad.gz - agent: device_script.sh local: ./device_script.sh + - agent: env_setup.sh + local: ./env_setup.sh test_cmds: | set -e # get hw-cert-team helper tools @@ -51,7 +53,7 @@ test_data: # Run the test script, but always retrieve artifacts even if it fails set +e - _run '/home/ubuntu/device_script.sh' + _run 'source /home/ubuntu/env_setup.sh && /home/ubuntu/device_script.sh' TEST_SCRIPT_EXIT=$? set -e diff --git a/.github/workflows/integration_tests.yaml.yml b/.github/workflows/integration_tests.yaml.yml deleted file mode 100644 index 7c27f248..00000000 --- a/.github/workflows/integration_tests.yaml.yml +++ /dev/null @@ -1,219 +0,0 @@ -name: Run integration tests on a DUT via testflinger -on: - workflow_dispatch: - pull_request: -jobs: - binary-integration-tests: - name: Build and Test the Daemon Binary on DUT - runs-on: [self-hosted, self-hosted-linux-amd64-jammy-private-endpoint-medium] - # todo: enable testflinger environment - # environment: - # name: testflinger - # if: ${{ vars.TESTFLINGER_TESTS_ENABLED == '1' && ! github.event.pull_request.draft}} - if: ${{ !github.event.pull_request.draft }} - steps: - - name: Checkout - uses: actions/checkout@v6 - - name: Install dependencies - shell: bash - run: | - echo "::group::install dependencies" - sudo snap install yq - echo "::endgroup::" - - name: Prepare Testflinger Job - id: prepare-tf-job - shell: bash - run: | - echo "::group::creating fpgad.gz tarball" - tar -czvf ../fpgad.gz --exclude="target" --exclude=".git" -C ../ ./fpgad - mv ../fpgad.gz ./fpgad.gz - echo "::endgroup::" - - echo "::group::creating job.yaml" - yq '.test_data.attachments = [ - {"agent": "device_script.sh", "local": "'"$(pwd)"'/.github/testflinger-assets/test_binary_device_script.sh"}, - {"agent": "fpgad.gz", "local": "'"$(pwd)"'/fpgad.gz"} - ]' .github/testflinger-assets/testflinger_job.yaml | tee job.yaml - echo "::endgroup::" - - echo "test_path=job.yaml" >> $GITHUB_OUTPUT - - name: Submit TF job - id: submit - uses: canonical/testflinger/.github/actions/submit@rev263 - continue-on-error: true - with: - poll: true - job-path: ${{ steps.prepare-tf-job.outputs.test_path }} - - name: Fetch and Display Artifacts - shell: bash - run: | - echo "::group::retrieve llvm-cov artifacts" - testflinger artifacts ${{steps.submit.outputs.id }} - tar -xvzf artifacts.tgz - echo "::endgroup::" - - for f in artifacts/* ; do - echo "::group::$f" - cat "$f" - echo "::endgroup::" - done - - name: Upload artifacts - uses: actions/upload-artifact@v7 - with: - name: llvm-cov-artifacts - path: artifacts/* - - name: Report test status - shell: bash - run: | - if [ "${{ steps.submit.outcome }}" != "success" ]; then - echo "::error::test job didn't complete successfully" - exit 1 - fi - if [[ ! -f artifacts/coverage_test.log ]]; then - echo "::error::coverage_test.log not found, did the tests fail to start?" - exit 1 - fi - if grep -q "test result: FAILED" artifacts/coverage_test.log; then - echo "::error::found failure phrase in coverage_test.log" - exit 1 - fi - snap-build: - name: Check Snap Builds - runs-on: [self-hosted, linux, noble, ARM64] - outputs: - artifact_id: ${{ steps.upload-snap.outputs.artifact-id }} - steps: - - uses: actions/checkout@v6 - - uses: canonical/action-build@v1.3.0 - id: snapcraft - - name: Package snap and components - id: package-snap - run: | - echo "::group::Listing snap and component files" - echo "Snap files:" - find . -maxdepth 1 -name "*.snap" -type f - echo "" - echo "Component files:" - find . -maxdepth 1 -name "*.comp" -type f - echo "::endgroup::" - - echo "::group::Packaging snap and component files" - # Collect snap file - SNAP_FILE="${{ steps.snapcraft.outputs.snap }}" - - # Find component files - must exist - COMP_FILES=$(find . -maxdepth 1 -name "*.comp" -type f) - if [ -z "$COMP_FILES" ]; then - echo "::error::No component files found - build may have failed" - exit 1 - fi - - # Create tarball with snap and components - tar -czvf fpgad-snap-package.tar.gz "$SNAP_FILE" $COMP_FILES - echo "Created tarball: fpgad-snap-package.tar.gz" - echo "package_path=$(pwd)/fpgad-snap-package.tar.gz" >> $GITHUB_OUTPUT - echo "::endgroup::" - - uses: actions/upload-artifact@v7 - id: upload-snap - with: - name: snap-fpgad-${{ github.event.pull_request != null && github.event.pull_request.head.sha || 'workflow-dispatch' }} - path: ${{ steps.package-snap.outputs.package_path }} - snap-integration-tests: - name: Install and Test the Snap Package on DUT - runs-on: [self-hosted, self-hosted-linux-amd64-noble-private-endpoint-medium] - if: ${{ !github.event.pull_request.draft }} - needs: ['snap-build'] - timeout-minutes: 3600 - steps: - - name: checkout - uses: actions/checkout@v6 - - name: Install dependencies - shell: bash - run: | - echo "::group::install dependencies" - sudo snap install yq - echo "::endgroup::" - - name: Download built snap - id: download-snap - uses: actions/download-artifact@v8 - with: - artifact-ids: ${{ needs.snap-build.outputs.artifact_id }} - path: ./snap-download - - name: Prepare Testflinger Job - id: prepare-tf-job - run: | - echo "::group::Locating snap package tarball" - # actions/download-artifact@v8 creates a subdirectory named after the artifact - # Look for the tarball in any subdirectory (artifact name may vary based on commit SHA) - echo "Contents of snap-download directory:" - ls -lR ./snap-download || true - - SNAP_TARBALL=$(find ./snap-download -name "fpgad-snap-package.tar.gz" -type f | head -n 1) - if [[ -z "$SNAP_TARBALL" || ! -f "$SNAP_TARBALL" ]]; then - echo "::error::Snap package tarball 'fpgad-snap-package.tar.gz' not found in download directory" - exit 1 - fi - echo "Found snap package at: $SNAP_TARBALL" - # Copy to current directory for use in job.yaml - cp "$SNAP_TARBALL" ./fpgad-snap-package.tar.gz - echo "Copied to: $(pwd)/fpgad-snap-package.tar.gz" - echo "::endgroup::" - - echo "::group::Create test_data tarball" - tar -czvf test_data.gz -C daemon/tests/test_data/ . - echo "::endgroup::" - - echo "::group::Create tests tarball" - tar -czvf tests.gz -C tests/ . - echo "::endgroup::" - - echo "::group::creating job.yaml" - yq '.test_data.attachments = [ - { "agent": "device_script.sh", "local": "'"$(pwd)"'/.github/testflinger-assets/test_snap_device_script.sh" }, - { "agent": "fpgad-snap-package.tar.gz", "local": "'"$(pwd)"'/fpgad-snap-package.tar.gz" }, - { "agent": "test_data.gz", "local": "'"$(pwd)"'/test_data.gz" }, - { "agent": "tests.gz", "local": "'"$(pwd)"'/tests.gz" } - ]' .github/testflinger-assets/testflinger_job.yaml | tee job.yaml - echo "::endgroup::" - - echo "test_path=job.yaml" >> $GITHUB_OUTPUT - - name: Submit TF job - id: submit - uses: canonical/testflinger/.github/actions/submit@rev263 - continue-on-error: true - with: - poll: true - job-path: ${{ steps.prepare-tf-job.outputs.test_path }} - - name: Fetch and Display Artifacts - shell: bash - run: | - echo "::group::retrieving snap test artifacts" - testflinger artifacts ${{steps.submit.outputs.id }} - tar -xvzf artifacts.tgz - echo "::endgroup::" - - for f in artifacts/* ; do - echo "::group::$f" - cat "$f" - echo "::endgroup::" - done - - name: Upload artifacts - uses: actions/upload-artifact@v7 - with: - name: snap-testing-artifacts - path: artifacts/* - - name: Report test status - shell: bash - run: |- - if [ "${{ steps.submit.outcome }}" != "success" ]; then - echo "::error::test job didn't complete successfully" - exit 1 - fi - if [[ ! -f artifacts/snap_test.log ]]; then - echo "::error::snap_test.log not found, did the tests fail to start?" - exit 1 - fi - if grep -q "FAILED (" artifacts/snap_test.log; then - echo "::error::found failure phrase in snap_test.log" - exit 1 - fi diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml new file mode 100644 index 00000000..f2222629 --- /dev/null +++ b/.github/workflows/integration_tests.yml @@ -0,0 +1,80 @@ +name: Run integration tests on a DUT via testflinger +on: + workflow_dispatch: + pull_request: +jobs: + binary-integration-tests: + name: Build and Test the Daemon Binary on DUT + runs-on: [self-hosted, self-hosted-linux-amd64-jammy-private-endpoint-medium] + # todo: enable testflinger environment + # environment: + # name: testflinger + # if: ${{ vars.TESTFLINGER_TESTS_ENABLED == '1' && ! github.event.pull_request.draft}} + if: ${{ !github.event.pull_request.draft }} + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Install dependencies + shell: bash + run: | + echo "::group::install dependencies" + sudo snap install yq + echo "::endgroup::" + - name: Prepare Testflinger Job + id: prepare-tf-job + shell: bash + run: | + echo "::group::creating fpgad.gz tarball" + tar -czvf ../fpgad.gz --exclude="target" --exclude=".git" -C ../ ./fpgad + mv ../fpgad.gz ./fpgad.gz + echo "::endgroup::" + + echo "::group::creating job.yaml" + yq '.test_data.attachments = [ + {"agent": "device_script.sh", "local": "'"$(pwd)"'/.github/testflinger-assets/test_binary_device_script.sh"}, + {"agent": "fpgad.gz", "local": "'"$(pwd)"'/fpgad.gz"}, + {"agent": "env_setup.sh", "local": "'"$(pwd)"'/.github/testflinger-assets/env_setup.sh"} + ]' .github/testflinger-assets/testflinger_job.yaml | tee job.yaml + echo "::endgroup::" + + echo "test_path=job.yaml" >> $GITHUB_OUTPUT + - name: Submit TF job + id: submit + uses: canonical/testflinger/.github/actions/submit@rev263 + continue-on-error: true + with: + poll: true + job-path: ${{ steps.prepare-tf-job.outputs.test_path }} + - name: Fetch and Display Artifacts + shell: bash + run: | + echo "::group::retrieve llvm-cov artifacts" + testflinger artifacts ${{steps.submit.outputs.id }} + tar -xvzf artifacts.tgz + echo "::endgroup::" + + for f in artifacts/* ; do + echo "::group::$f" + cat "$f" + echo "::endgroup::" + done + - name: Upload artifacts + uses: actions/upload-artifact@v7 + with: + name: llvm-cov-artifacts + path: artifacts/* + - name: Report test status + shell: bash + run: | + if [ "${{ steps.submit.outcome }}" != "success" ]; then + echo "::error::test job didn't complete successfully" + exit 1 + fi + if [[ ! -f artifacts/coverage_test.log ]]; then + echo "::error::coverage_test.log not found, did the tests fail to start?" + exit 1 + fi + if grep -q "test result: FAILED" artifacts/coverage_test.log; then + echo "::error::found failure phrase in coverage_test.log" + exit 1 + fi diff --git a/.github/workflows/snap_integration_tests.yml b/.github/workflows/snap_integration_tests.yml new file mode 100644 index 00000000..a4d1b909 --- /dev/null +++ b/.github/workflows/snap_integration_tests.yml @@ -0,0 +1,151 @@ +name: Run snap integration tests on a DUT via testflinger +on: + workflow_dispatch: + inputs: + snap_test_source: + description: "Snap source channel when source=store (examples: local, beta, edge, stable). If source=local, will use a locally built snap." + required: true + default: local + type: string + tf_queue: + description: "Device identifier suffix as seen in C3 (example: xilinx-kv260-c36973)" + required: true + default: xilinx-kv260-c36973 + type: string + pull_request: +jobs: + snap-build: + name: Check Snap Builds + runs-on: [self-hosted, linux, noble, ARM64] + outputs: + artifact_id: ${{ steps.upload-snap.outputs.artifact-id }} + steps: + - uses: actions/checkout@v6 + - uses: canonical/action-build@v1.3.0 + id: snapcraft + if: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.snap_test_source == 'local') }} + - uses: actions/upload-artifact@v6 + id: upload-snap + if: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.snap_test_source == 'local') }} + with: + name: snap-fpgad-${{ github.event.pull_request != null && github.event.pull_request.head.sha || 'workflow-dispatch' }} + path: ${{ steps.snapcraft.outputs.snap }} + snap-integration-tests: + name: Install and Test the Snap Package on DUT + runs-on: [self-hosted, self-hosted-linux-amd64-noble-private-endpoint-medium] + if: ${{ github.event_name == 'workflow_dispatch' || !github.event.pull_request.draft }} + needs: ['snap-build'] + timeout-minutes: 3600 + steps: + - name: checkout + uses: actions/checkout@v6 + - name: Install dependencies + shell: bash + run: | + echo "::group::install dependencies" + sudo snap install yq + echo "::endgroup::" + - name: Download built snap + id: download-snap + uses: actions/download-artifact@v7 + if: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.snap_test_source == 'local') }} + with: + artifact-ids: ${{ needs['snap-build'].outputs.artifact_id }} + path: ./ + - name: Prepare Testflinger Job + id: prepare-tf-job + env: + ROOT_DIR: ${{ github.workspace }} + SNAP_TEST_SOURCE: ${{ inputs.snap_test_source || 'local' }} + TF_JOB_QUEUE: ${{ inputs.tf_queue || 'xilinx-kv260-c36973' }} + shell: bash + run: | + cat > "${ROOT_DIR}/.github/testflinger-assets/env_setup.sh" < job.yaml + + # Append source-specific attachment + if [[ "${SNAP_TEST_SOURCE}" == "local" ]]; then + yq -i '.test_data.attachments += [ + { "agent": "fpgad.snap", "local": strenv(ROOT_DIR) + "/fpgad.snap" } + ]' job.yaml + fi + + cat job.yaml + echo "::endgroup::" + + echo "test_path=job.yaml" >> $GITHUB_OUTPUT + - name: Submit TF job + id: submit + uses: canonical/testflinger/.github/actions/submit@rev263 + continue-on-error: true + with: + poll: true + job-path: ${{ steps.prepare-tf-job.outputs.test_path }} + - name: Fetch and Display Artifacts + shell: bash + run: | + echo "::group::retrieving snap test artifacts" + testflinger artifacts ${{steps.submit.outputs.id }} + tar -xvzf artifacts.tgz + echo "::endgroup::" + for f in artifacts/* ; do + echo "::group::$f" + cat "$f" + echo "::endgroup::" + done + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + name: snap-testing-artifacts + path: artifacts/* + - name: Report test status + shell: bash + run: |- + if [ "${{ steps.submit.outcome }}" != "success" ]; then + echo "::error::test job didn't complete successfully" + exit 1 + fi + if [[ ! -f artifacts/snap_test.log ]]; then + echo "::error::snap_test.log not found, did the tests fail to start?" + exit 1 + fi + if grep -q "FAILED (" artifacts/snap_test.log; then + echo "::error::found failure phrase in snap_test.log" + exit 1 + fi