diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 72712747f..2b19818e2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -28,8 +28,10 @@ env: binutils dpkg-dev gcc gdb kmod libc6-dev libglib2.0-dev libxml2-utils polkitd python3-systemd valgrind xterm system_deps: > - chaos-marmosets dbus dbus-x11 dirmngr dpkg-dev gcc gdb gvfs-daemons psmisc - python3-dbus ubuntu-dbgsym-keyring ubuntu-keyring valgrind xterm xvfb + dbus dbus-x11 gdb gvfs-daemons psmisc python3-dbus + ubuntu-dbgsym-keyring ubuntu-keyring xterm xvfb + system_internet_deps: > + chaos-marmosets dpkg-dev gdb ubuntu-dbgsym-keyring ubuntu-keyring valgrind jobs: linter: @@ -126,7 +128,7 @@ jobs: - name: Run unit and integration tests run: > WORKDIR=$(mktemp -d -t apport.XXXXXXXXXX) - && cp -r tests "$WORKDIR" + && cp -r tests pyproject.toml "$WORKDIR" && cd "$WORKDIR" && python3 -m pytest -v -ra --cov=$(pwd) --cov-report=xml --durations=0 tests/unit/ tests/integration/ @@ -162,8 +164,8 @@ jobs: - uses: actions/checkout@v6 - name: Run all tests (to check if they are skipped or succeed) run: > - SKIP_ONLINE_TESTS=1 python3 -m pytest -v -ra --cov=$(pwd) - --cov-report=xml --durations=0 tests/ + python3 -m pytest -v -ra --cov=$(pwd) + --cov-report=xml --durations=0 -m "not requires_internet" tests/ - name: Upload coverage uses: actions/upload-artifact@v7 with: @@ -182,18 +184,11 @@ jobs: # - ubuntu:resolute container: image: ${{ matrix.container }} - options: --cap-add=SYS_PTRACE --security-opt seccomp=unconfined steps: - name: Sanitize container name (for artifact name) run: | container=$(echo "${{ matrix.container }}" | sed 's/:/-/') echo "JOB=${GITHUB_JOB}-${container}" >> "$GITHUB_ENV" - - name: Enable 'deb-src' URIs in /etc/apt/sources.list - run: > - sed -i '/^#\sdeb-src /s/^#\s//' /etc/apt/sources.list - && ! test -e /etc/apt/sources.list.d/ubuntu.sources - || sed -i 's/^Types:.*/Types: deb deb-src/g' - /etc/apt/sources.list.d/ubuntu.sources - name: Install dependencies run: > apt-get update @@ -202,12 +197,13 @@ jobs: - uses: actions/checkout@v6 - name: Start D-Bus daemon run: mkdir -p /run/dbus && dbus-daemon --system --fork - - name: Run system tests + - name: Run system tests that do not require Internet access env: GDK_BACKEND: x11 run: > xvfb-run python3 -m pytest -v -ra --cov=$(pwd) - --cov-report=xml --durations=0 tests/system/ + --cov-report=xml --durations=0 + -m "not requires_internet" tests/system/ - name: Stop D-Bus daemon run: kill $(cat /run/dbus/pid) - name: Upload coverage @@ -217,6 +213,87 @@ jobs: path: ./coverage.xml system-installed: + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + steps: + - name: Remove system installed Apport + run: > + sudo apt-get remove --purge --yes + apport python3-apport python3-problem-report + - name: Install dependencies + run: > + sudo apt-get update + && sudo apt-get install --no-install-recommends --yes + $base_deps $build_deps $common_deps $gtk_deps $kde_deps $system_deps + - uses: actions/checkout@v6 + - name: Install + run: > + sudo python3 -m coverage run + ./setup.py install --install-layout=deb --prefix=/usr --root=/ + && sudo chown "$SUDO_UID:$SUDO_GID" .coverage + && python3 -m coverage xml -o coverage-setup.xml + - name: Cleanup /var/crash/ + run: sudo rm -f /var/crash/*.crash + - name: Enable Apport + run: sudo /usr/share/apport/apport --start + - name: Run system tests that do not require Internet access + env: + GDK_BACKEND: x11 + run: > + WORKDIR=$(mktemp -d -t apport.XXXXXXXXXX) + && cp -r tests pyproject.toml "$WORKDIR" + && cd "$WORKDIR" + && sudo xvfb-run python3 -m pytest -v -ra --cov=$(pwd) + --cov-report=xml --durations=0 + -m "not requires_internet" tests/system/ + && cd - + && cp "$WORKDIR/coverage.xml" . + - name: Upload coverage + uses: actions/upload-artifact@v7 + with: + name: coverage-${{ github.job }} + path: ./coverage*.xml + + system-internet: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + container: + - ubuntu:noble + - ubuntu:questing + - ubuntu:resolute + container: + image: ${{ matrix.container }} + steps: + - name: Sanitize container name (for artifact name) + run: | + container=$(echo "${{ matrix.container }}" | sed 's/:/-/') + echo "JOB=${GITHUB_JOB}-${container}" >> "$GITHUB_ENV" + - name: Enable 'deb-src' URIs in /etc/apt/sources.list + run: > + sed -i '/^#\sdeb-src /s/^#\s//' /etc/apt/sources.list + && ! test -e /etc/apt/sources.list.d/ubuntu.sources + || sed -i 's/^Types:.*/Types: deb deb-src/g' + /etc/apt/sources.list.d/ubuntu.sources + - name: Install dependencies + run: > + apt-get update + && apt-get install --no-install-recommends --yes + $base_deps $common_deps $system_internet_deps + - uses: actions/checkout@v6 + - name: Run system tests that require Internet access + run: > + python3 -m pytest -v -ra --cov=$(pwd) + --cov-report=xml --durations=0 -m "requires_internet" tests/system/ + - name: Upload coverage + uses: actions/upload-artifact@v7 + with: + name: coverage-${{ env.JOB }} + path: ./coverage.xml + + system-internet-installed: runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -233,7 +310,7 @@ jobs: run: > sudo apt-get update && sudo apt-get install --no-install-recommends --yes - $base_deps $build_deps $common_deps $gtk_deps $kde_deps $system_deps + $base_deps $build_deps $common_deps $gtk_deps $system_internet_deps - uses: actions/checkout@v6 - name: Install run: > @@ -245,15 +322,15 @@ jobs: run: sudo rm -f /var/crash/*.crash - name: Enable Apport run: sudo /usr/share/apport/apport --start - - name: Run system tests + - name: Run system tests that require Internet access env: GDK_BACKEND: x11 run: > WORKDIR=$(mktemp -d -t apport.XXXXXXXXXX) - && cp -r tests "$WORKDIR" + && cp -r tests pyproject.toml "$WORKDIR" && cd "$WORKDIR" && sudo xvfb-run python3 -m pytest -v -ra --cov=$(pwd) - --cov-report=xml --durations=0 tests/system/ + --cov-report=xml --durations=0 -m "requires_internet" tests/system/ && cd - && cp "$WORKDIR/coverage.xml" . - name: Upload coverage @@ -282,6 +359,8 @@ jobs: - skip - system - system-installed + - system-internet + - system-internet-installed runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 diff --git a/pyproject.toml b/pyproject.toml index 33e18b526..a9e22709c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,9 @@ score = false [tool.pytest.ini_options] addopts = "--cov-branch" +markers = [ + "requires_internet: marks tests that require internet connection (deselect with '-m \"not requires_internet\"')", +] [tool.ruff] include = [ diff --git a/tests/README.md b/tests/README.md index ed4735602..d599b5ff7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -51,8 +51,8 @@ The [system directory](./system) contains system tests. It also contains integration tests that need special environment setup or have a long execution time. The GTK and KDE UI integration tests need a window system, which can be provided by `xvfb-run`. Some integration tests query https://launchpad.net/. -These tests can be skipped by setting the environment variable -`SKIP_ONLINE_TESTS` to something non empty. The test in +These tests can be filtered out with the pytest marker expression +`-m "not requires_internet"`. The test in [test_python_crashes.py](./system/test_python_crashes.py) need a running D-Bus daemon. Whit a D-Bus daemon running, the system tests can be run from the top directory by calling: diff --git a/tests/helper.py b/tests/helper.py index d294eaef8..add6176ed 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -1,7 +1,6 @@ """Helper functions for the test cases.""" import contextlib -import functools import importlib.machinery import importlib.util import os @@ -10,8 +9,6 @@ import subprocess import time import unittest.mock -import urllib.error -import urllib.request from collections.abc import Callable, Generator, Iterator, Sequence, Set from typing import Any from unittest.mock import MagicMock @@ -33,24 +30,6 @@ def get_init_system() -> str: return comm.read().rstrip() -@functools.lru_cache(maxsize=1) -def has_internet() -> bool: - """Return if there is sufficient network connection for the tests. - - This checks if https://api.launchpad.net/devel/ubuntu/ can be downloaded - from, to check if we can run the online tests. - """ - if os.environ.get("SKIP_ONLINE_TESTS"): - return False - try: - with urllib.request.urlopen( - "https://api.launchpad.net/devel/ubuntu/", timeout=30 - ) as url: - return b"web_link" in url.readline() - except urllib.error.URLError: - return False - - def import_module_from_file(path: pathlib.Path) -> Any: """Import a module by its filename.""" name = path.stem.replace("-", "_") diff --git a/tests/integration/test_crashdb_launchpad.py b/tests/integration/test_crashdb_launchpad.py index d89ebc9eb..555f28c21 100644 --- a/tests/integration/test_crashdb_launchpad.py +++ b/tests/integration/test_crashdb_launchpad.py @@ -25,6 +25,8 @@ from typing import TYPE_CHECKING, Any from unittest.mock import MagicMock +import pytest + try: from launchpadlib.errors import HTTPError @@ -58,6 +60,7 @@ def try_to_get_from_cache(*args: Any, **kwargs: Any) -> Any: return try_to_get_from_cache +@pytest.mark.requires_internet @unittest.skipIf(IMPORT_ERROR, f"Python module not available: {IMPORT_ERROR}") @unittest.skipUnless( "TEST_LAUNCHPAD" in os.environ, diff --git a/tests/system/test_apport_retrace.py b/tests/system/test_apport_retrace.py index 328a4bb05..7fc2fa72e 100644 --- a/tests/system/test_apport_retrace.py +++ b/tests/system/test_apport_retrace.py @@ -14,7 +14,6 @@ from apport.packaging_impl.apt_dpkg import impl from apport.report import Report -from tests.helper import has_internet from tests.paths import get_test_data_directory, local_test_environment CODENAME_DISTRO_RELEASE_MAP = {"jammy": "Ubuntu 22.04"} @@ -195,7 +194,7 @@ def _assert_cache_has_content( assert list((aptroot / "var/cache/apt/archives").iterdir()) -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet @pytest.mark.skipif( impl.get_system_architecture() == "s390x", reason="GDB has issues with divide-by-zero on s390x (LP: #2075204)", @@ -224,7 +223,7 @@ def test_retrace_system_sandbox( _assert_divide_by_zero_retrace(report) -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet @pytest.mark.skipif( impl.get_system_architecture() == "amd64", reason="GDB sandbox is only available on amd64", @@ -252,7 +251,7 @@ def test_retrace_system_sandbox_gdb_sandbox_nonamd64( assert "gdb sandboxes are only implemented for amd64 hosts" in ret.stderr -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet @pytest.mark.skipif( impl.get_system_architecture() != "amd64", reason="Testing the GDB sandbox erroring out on non-AMD64", @@ -282,7 +281,7 @@ def test_retrace_system_sandbox_gdb_sandbox( _assert_divide_by_zero_retrace(report) -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet @pytest.mark.skipif( impl.get_system_architecture() != "amd64" and shutil.which("gdb-multiarch") is None, reason="gdb-multiarch is needed for proper retracing on foreign architectures", @@ -315,7 +314,7 @@ def test_retrace_jammy_sandbox( _assert_cache_has_content(module_cachedir, "amd64", "jammy") -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet @pytest.mark.skipif( impl.get_system_architecture() != "amd64", reason="GDB sandbox only available on amd64", diff --git a/tests/system/test_apport_valgrind.py b/tests/system/test_apport_valgrind.py index d1f82913c..294ddcfe9 100644 --- a/tests/system/test_apport_valgrind.py +++ b/tests/system/test_apport_valgrind.py @@ -15,6 +15,8 @@ import tempfile import unittest +import pytest + from tests.helper import get_gnu_coreutils_cmd, skip_if_command_is_missing from tests.paths import local_test_environment @@ -43,6 +45,7 @@ def tearDown(self) -> None: shutil.rmtree(self.workdir) os.chdir(self.pwd) + @pytest.mark.requires_internet @unittest.skipIf(MEM_TOTAL_MiB < 2000, f"{MEM_TOTAL_MiB} MiB is not enough memory") def test_sandbox_cache_options(self) -> None: """apport-valgrind creates a user specified sandbox and cache""" diff --git a/tests/system/test_github_query.py b/tests/system/test_github_query.py index 8a41d6da4..7b46effe0 100644 --- a/tests/system/test_github_query.py +++ b/tests/system/test_github_query.py @@ -3,6 +3,8 @@ import unittest from unittest.mock import Mock +import pytest + import apport.crashdb_impl.github SOME_ID = "a654870577ad2a2ab5b1" @@ -18,6 +20,7 @@ def setUp(self) -> None: self.crashdb.app_id, self.message_cb ) + @pytest.mark.requires_internet def test_api_authentication(self) -> None: """Test if we can contact Github authentication service.""" with self.github as github: diff --git a/tests/system/test_packaging_apt_dpkg.py b/tests/system/test_packaging_apt_dpkg.py index fee721ffa..19d25cb3b 100644 --- a/tests/system/test_packaging_apt_dpkg.py +++ b/tests/system/test_packaging_apt_dpkg.py @@ -17,7 +17,6 @@ import pytest from apport.packaging_impl.apt_dpkg import _parse_deb822_sources, impl -from tests.helper import has_internet from tests.paths import get_test_data_directory if shutil.which("dpkg") is None: @@ -75,7 +74,7 @@ def reset_impl() -> Iterator[None]: impl.configuration = orig_conf -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_install_packages_versioned( configdir: str, cachedir: str, rootdir: str, apt_style: AptStyle ) -> None: @@ -217,7 +216,7 @@ def sandbox_ver(pkg: str) -> str: assert os.path.exists(os.path.join(rootdir, "usr/bin/dpkg")) -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_install_packages_unversioned( configdir: str, cachedir: str, rootdir: str, apt_style: AptStyle ) -> None: @@ -267,7 +266,7 @@ def test_install_packages_unversioned( assert len(pkglist), str(pkglist) == 3 -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_install_packages_dependencies( configdir: str, rootdir: str, apt_style: AptStyle ) -> None: @@ -294,7 +293,7 @@ def test_install_packages_dependencies( assert "libc6" not in result -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_install_packages_system( cachedir: str, workdir: str, rootdir: str, apt_style: AptStyle ) -> None: @@ -347,7 +346,7 @@ def test_install_packages_system( assert os.path.exists(os.path.join(rootdir, "usr/bin/gzip")) -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_install_packages_error( configdir: str, cachedir: str, rootdir: str, apt_style: AptStyle ) -> None: @@ -381,7 +380,7 @@ def test_install_packages_error( ) -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_install_packages_permanent_sandbox( configdir: str, cachedir: str, rootdir: str, apt_style: AptStyle ) -> None: @@ -490,7 +489,7 @@ def test_install_packages_permanent_sandbox( apt_pkg.config.set("Acquire::http::Proxy", orig_apt_proxy) -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_install_packages_permanent_sandbox_repack( configdir: str, cachedir: str, rootdir: str, apt_style: AptStyle ) -> None: @@ -532,7 +531,7 @@ def test_install_packages_permanent_sandbox_repack( assert os.readlink(curl_library) == "libcurl-gnutls.so" -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet @pytest.mark.skipif( impl.get_system_architecture() == "armhf", reason="native armhf architecture" ) @@ -570,7 +569,7 @@ def test_install_packages_armhf( assert f"libc6_{got_version}_armhf.deb" in cache -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_install_packages_from_launchpad( configdir: str, cachedir: str, rootdir: str, apt_style: AptStyle ) -> None: @@ -634,7 +633,7 @@ def sandbox_ver(pkg: str, debian: bool = True) -> str: assert ("qemu-utils-dbgsym", _strip_epoch(wanted["qemu-utils"])) in cache_versions -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_install_old_packages( configdir: str, cachedir: str, rootdir: str, apt_style: AptStyle ) -> None: @@ -678,7 +677,7 @@ def sandbox_ver(pkg: str) -> str: assert f"{wanted_package} {wanted_version}" in pkglist -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_get_source_tree_sandbox( configdir: str, workdir: str, rootdir: str, apt_style: AptStyle ) -> None: @@ -697,7 +696,7 @@ def test_get_source_tree_sandbox( assert res.endswith("/base-files-12ubuntu4") -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_get_source_tree_lp_sandbox( configdir: str, workdir: str, rootdir: str, apt_style: AptStyle ) -> None: @@ -721,7 +720,7 @@ def test_get_source_tree_lp_sandbox( assert res.endswith(f"/{wanted_package}-{upstream_version}") -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_create_sources_for_a_named_ppa( configdir: str, rootdir: str, apt_style: AptStyle ) -> None: @@ -760,7 +759,7 @@ def test_create_sources_for_a_named_ppa( assert expected_key == actual_key -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_create_sources_for_an_unnamed_ppa( configdir: str, rootdir: str, apt_style: AptStyle ) -> None: @@ -844,7 +843,7 @@ def test_use_sources_for_a_ppa( ] -@pytest.mark.skipif(not has_internet(), reason="online test") +@pytest.mark.requires_internet def test_install_package_from_a_ppa( configdir: str, cachedir: str, rootdir: str, apt_style: AptStyle ) -> None: