ci: run Internet requiring system tests separately

.github/workflows/ci.yaml

@@ -28,8 +28,10 @@ env:
     binutils dpkg-dev gcc gdb kmod libc6-dev libglib2.0-dev libxml2-utils
     polkitd python3-systemd valgrind xterm
   system_deps: >
-    chaos-marmosets dbus dbus-x11 dirmngr dpkg-dev gcc gdb gvfs-daemons psmisc
-    python3-dbus ubuntu-dbgsym-keyring ubuntu-keyring valgrind xterm xvfb
+    dbus dbus-x11 gdb gvfs-daemons psmisc python3-dbus
+    ubuntu-dbgsym-keyring ubuntu-keyring xterm xvfb
+  system_internet_deps: >
+    chaos-marmosets dpkg-dev gdb ubuntu-dbgsym-keyring ubuntu-keyring valgrind
 
 jobs:
   linter:
@@ -126,7 +128,7 @@ jobs:
       - name: Run unit and integration tests
         run: >
           WORKDIR=$(mktemp -d -t apport.XXXXXXXXXX)
-          && cp -r tests "$WORKDIR"
+          && cp -r tests pyproject.toml "$WORKDIR"
           && cd "$WORKDIR"
           && python3 -m pytest -v -ra --cov=$(pwd) --cov-report=xml
           --durations=0 tests/unit/ tests/integration/
@@ -162,8 +164,8 @@ jobs:
       - uses: actions/checkout@v6
       - name: Run all tests (to check if they are skipped or succeed)
         run: >
-          SKIP_ONLINE_TESTS=1 python3 -m pytest -v -ra --cov=$(pwd)
-          --cov-report=xml --durations=0 tests/
+          python3 -m pytest -v -ra --cov=$(pwd)
+          --cov-report=xml --durations=0 -m "not requires_internet" tests/
       - name: Upload coverage
         uses: actions/upload-artifact@v7
         with:
@@ -182,18 +184,11 @@ jobs:
           # - ubuntu:resolute
     container:
       image: ${{ matrix.container }}
-      options: --cap-add=SYS_PTRACE --security-opt seccomp=unconfined
     steps:
       - name: Sanitize container name (for artifact name)
         run: |
           container=$(echo "${{ matrix.container }}" | sed 's/:/-/')
           echo "JOB=${GITHUB_JOB}-${container}" >> "$GITHUB_ENV"
-      - name: Enable 'deb-src' URIs in /etc/apt/sources.list
-        run: >
-          sed -i '/^#\sdeb-src /s/^#\s//' /etc/apt/sources.list
-          && ! test -e /etc/apt/sources.list.d/ubuntu.sources
-          || sed -i 's/^Types:.*/Types: deb deb-src/g'
-          /etc/apt/sources.list.d/ubuntu.sources
       - name: Install dependencies
         run: >
           apt-get update
@@ -202,12 +197,13 @@ jobs:
       - uses: actions/checkout@v6
       - name: Start D-Bus daemon
         run: mkdir -p /run/dbus && dbus-daemon --system --fork
-      - name: Run system tests
+      - name: Run system tests that do not require Internet access
         env:
           GDK_BACKEND: x11
         run: >
           xvfb-run python3 -m pytest -v -ra --cov=$(pwd)
-          --cov-report=xml --durations=0 tests/system/
+          --cov-report=xml --durations=0
+          -m "not requires_internet" tests/system/
       - name: Stop D-Bus daemon
         run: kill $(cat /run/dbus/pid)
       - name: Upload coverage
@@ -217,6 +213,87 @@ jobs:
           path: ./coverage.xml
 
   system-installed:
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Remove system installed Apport
+        run: >
+          sudo apt-get remove --purge --yes
+          apport python3-apport python3-problem-report
+      - name: Install dependencies
+        run: >
+          sudo apt-get update
+          && sudo apt-get install --no-install-recommends --yes
+          $base_deps $build_deps $common_deps $gtk_deps $kde_deps $system_deps
+      - uses: actions/checkout@v6
+      - name: Install
+        run: >
+          sudo python3 -m coverage run
+          ./setup.py install --install-layout=deb --prefix=/usr --root=/
+          && sudo chown "$SUDO_UID:$SUDO_GID" .coverage
+          && python3 -m coverage xml -o coverage-setup.xml
+      - name: Cleanup /var/crash/
+        run: sudo rm -f /var/crash/*.crash
+      - name: Enable Apport
+        run: sudo /usr/share/apport/apport --start
+      - name: Run system tests that do not require Internet access
+        env:
+          GDK_BACKEND: x11
+        run: >
+          WORKDIR=$(mktemp -d -t apport.XXXXXXXXXX)
+          && cp -r tests pyproject.toml "$WORKDIR"
+          && cd "$WORKDIR"
+          && sudo xvfb-run python3 -m pytest -v -ra --cov=$(pwd)
+          --cov-report=xml --durations=0
+          -m "not requires_internet" tests/system/
+          && cd -
+          && cp "$WORKDIR/coverage.xml" .
+      - name: Upload coverage
+        uses: actions/upload-artifact@v7
+        with:
+          name: coverage-${{ github.job }}
+          path: ./coverage*.xml
+
+  system-internet:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        container:
+          - ubuntu:noble
+          - ubuntu:questing
+          - ubuntu:resolute
+    container:
+      image: ${{ matrix.container }}
+    steps:
+      - name: Sanitize container name (for artifact name)
+        run: |
+          container=$(echo "${{ matrix.container }}" | sed 's/:/-/')
+          echo "JOB=${GITHUB_JOB}-${container}" >> "$GITHUB_ENV"
+      - name: Enable 'deb-src' URIs in /etc/apt/sources.list
+        run: >
+          sed -i '/^#\sdeb-src /s/^#\s//' /etc/apt/sources.list
+          && ! test -e /etc/apt/sources.list.d/ubuntu.sources
+          || sed -i 's/^Types:.*/Types: deb deb-src/g'
+          /etc/apt/sources.list.d/ubuntu.sources
+      - name: Install dependencies
+        run: >
+          apt-get update
+          && apt-get install --no-install-recommends --yes
+          $base_deps $common_deps $system_internet_deps
+      - uses: actions/checkout@v6
+      - name: Run system tests that require Internet access
+        run: >
+          python3 -m pytest -v -ra --cov=$(pwd)
+          --cov-report=xml --durations=0 -m "requires_internet" tests/system/
+      - name: Upload coverage
+        uses: actions/upload-artifact@v7
+        with:
+          name: coverage-${{ env.JOB }}
+          path: ./coverage.xml
+
+  system-internet-installed:
     runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
@@ -233,7 +310,7 @@ jobs:
         run: >
           sudo apt-get update
           && sudo apt-get install --no-install-recommends --yes
-          $base_deps $build_deps $common_deps $gtk_deps $kde_deps $system_deps
+          $base_deps $build_deps $common_deps $gtk_deps $system_internet_deps
       - uses: actions/checkout@v6
       - name: Install
         run: >
@@ -245,15 +322,15 @@ jobs:
         run: sudo rm -f /var/crash/*.crash
       - name: Enable Apport
         run: sudo /usr/share/apport/apport --start
-      - name: Run system tests
+      - name: Run system tests that require Internet access
         env:
           GDK_BACKEND: x11
         run: >
           WORKDIR=$(mktemp -d -t apport.XXXXXXXXXX)
-          && cp -r tests "$WORKDIR"
+          && cp -r tests pyproject.toml "$WORKDIR"
           && cd "$WORKDIR"
           && sudo xvfb-run python3 -m pytest -v -ra --cov=$(pwd)
-          --cov-report=xml --durations=0 tests/system/
+          --cov-report=xml --durations=0 -m "requires_internet" tests/system/
           && cd -
           && cp "$WORKDIR/coverage.xml" .
       - name: Upload coverage
@@ -282,6 +359,8 @@ jobs:
       - skip
       - system
       - system-installed
+      - system-internet
+      - system-internet-installed
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6

pyproject.toml

@@ -111,6 +111,9 @@ score = false
 
 [tool.pytest.ini_options]
 addopts = "--cov-branch"
+markers = [
+    "requires_internet: marks tests that require internet connection (deselect with '-m \"not requires_internet\"')",
+]
 
 [tool.ruff]
 include = [

tests/README.md

@@ -51,8 +51,8 @@ The [system directory](./system) contains system tests. It also contains
 integration tests that need special environment setup or have a long execution
 time. The GTK and KDE UI integration tests need a window system, which can be
 provided by `xvfb-run`. Some integration tests query https://launchpad.net/.
-These tests can be skipped by setting the environment variable
-`SKIP_ONLINE_TESTS` to something non empty. The test in
+These tests can be filtered out with the pytest marker expression
+`-m "not requires_internet"`. The test in
 [test_python_crashes.py](./system/test_python_crashes.py) need a running D-Bus
 daemon. Whit a D-Bus daemon running, the system tests can be run from the top
 directory by calling:

tests/helper.py

@@ -1,7 +1,6 @@
 """Helper functions for the test cases."""
 
 import contextlib
-import functools
 import importlib.machinery
 import importlib.util
 import os
@@ -10,8 +9,6 @@
 import subprocess
 import time
 import unittest.mock
-import urllib.error
-import urllib.request
 from collections.abc import Callable, Generator, Iterator, Sequence, Set
 from typing import Any
 from unittest.mock import MagicMock
@@ -33,24 +30,6 @@ def get_init_system() -> str:
         return comm.read().rstrip()
 
 
-@functools.lru_cache(maxsize=1)
-def has_internet() -> bool:
-    """Return if there is sufficient network connection for the tests.
-
-    This checks if https://api.launchpad.net/devel/ubuntu/ can be downloaded
-    from, to check if we can run the online tests.
-    """
-    if os.environ.get("SKIP_ONLINE_TESTS"):
-        return False
-    try:
-        with urllib.request.urlopen(
-            "https://api.launchpad.net/devel/ubuntu/", timeout=30
-        ) as url:
-            return b"web_link" in url.readline()
-    except urllib.error.URLError:
-        return False
-
-
 def import_module_from_file(path: pathlib.Path) -> Any:
     """Import a module by its filename."""
     name = path.stem.replace("-", "_")

tests/integration/test_crashdb_launchpad.py

@@ -25,6 +25,8 @@
 from typing import TYPE_CHECKING, Any
 from unittest.mock import MagicMock
 
+import pytest
+
 try:
     from launchpadlib.errors import HTTPError
 
@@ -58,6 +60,7 @@ def try_to_get_from_cache(*args: Any, **kwargs: Any) -> Any:
     return try_to_get_from_cache
 
 
+@pytest.mark.requires_internet
 @unittest.skipIf(IMPORT_ERROR, f"Python module not available: {IMPORT_ERROR}")
 @unittest.skipUnless(
     "TEST_LAUNCHPAD" in os.environ,

tests/system/test_apport_retrace.py

@@ -14,7 +14,6 @@
 
 from apport.packaging_impl.apt_dpkg import impl
 from apport.report import Report
-from tests.helper import has_internet
 from tests.paths import get_test_data_directory, local_test_environment
 
 CODENAME_DISTRO_RELEASE_MAP = {"jammy": "Ubuntu 22.04"}
@@ -195,7 +194,7 @@ def _assert_cache_has_content(
     assert list((aptroot / "var/cache/apt/archives").iterdir())
 
 
-@pytest.mark.skipif(not has_internet(), reason="online test")
+@pytest.mark.requires_internet
 @pytest.mark.skipif(
     impl.get_system_architecture() == "s390x",
     reason="GDB has issues with divide-by-zero on s390x (LP: #2075204)",
@@ -224,7 +223,7 @@ def test_retrace_system_sandbox(
     _assert_divide_by_zero_retrace(report)
 
 
-@pytest.mark.skipif(not has_internet(), reason="online test")
+@pytest.mark.requires_internet
 @pytest.mark.skipif(
     impl.get_system_architecture() == "amd64",
     reason="GDB sandbox is only available on amd64",
@@ -252,7 +251,7 @@ def test_retrace_system_sandbox_gdb_sandbox_nonamd64(
     assert "gdb sandboxes are only implemented for amd64 hosts" in ret.stderr
 
 
-@pytest.mark.skipif(not has_internet(), reason="online test")
+@pytest.mark.requires_internet
 @pytest.mark.skipif(
     impl.get_system_architecture() != "amd64",
     reason="Testing the GDB sandbox erroring out on non-AMD64",
@@ -282,7 +281,7 @@ def test_retrace_system_sandbox_gdb_sandbox(
     _assert_divide_by_zero_retrace(report)
 
 
-@pytest.mark.skipif(not has_internet(), reason="online test")
+@pytest.mark.requires_internet
 @pytest.mark.skipif(
     impl.get_system_architecture() != "amd64" and shutil.which("gdb-multiarch") is None,
     reason="gdb-multiarch is needed for proper retracing on foreign architectures",
@@ -315,7 +314,7 @@ def test_retrace_jammy_sandbox(
     _assert_cache_has_content(module_cachedir, "amd64", "jammy")
 
 
-@pytest.mark.skipif(not has_internet(), reason="online test")
+@pytest.mark.requires_internet
 @pytest.mark.skipif(
     impl.get_system_architecture() != "amd64",
     reason="GDB sandbox only available on amd64",

tests/system/test_apport_valgrind.py

@@ -15,6 +15,8 @@
 import tempfile
 import unittest
 
+import pytest
+
 from tests.helper import get_gnu_coreutils_cmd, skip_if_command_is_missing
 from tests.paths import local_test_environment
 
@@ -43,6 +45,7 @@ def tearDown(self) -> None:
         shutil.rmtree(self.workdir)
         os.chdir(self.pwd)
 
+    @pytest.mark.requires_internet
     @unittest.skipIf(MEM_TOTAL_MiB < 2000, f"{MEM_TOTAL_MiB} MiB is not enough memory")
     def test_sandbox_cache_options(self) -> None:
         """apport-valgrind creates a user specified sandbox and cache"""

tests/system/test_github_query.py

@@ -3,6 +3,8 @@
 import unittest
 from unittest.mock import Mock
 
+import pytest
+
 import apport.crashdb_impl.github
 
 SOME_ID = "a654870577ad2a2ab5b1"
@@ -18,6 +20,7 @@ def setUp(self) -> None:
             self.crashdb.app_id, self.message_cb
         )
 
+    @pytest.mark.requires_internet
     def test_api_authentication(self) -> None:
         """Test if we can contact Github authentication service."""
         with self.github as github: