From 6c95486f00e4772573337de7edcfec07984fc0d0 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Mon, 23 Mar 2026 17:40:22 +0100 Subject: [PATCH 01/15] test/imgtestlib: codestyle fixes - E252 missing whitespace around parameter equals - E713 test for membership should be 'not in' - E302 expected 2 blank lines, found 1 --- test/scripts/imgtestlib.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/scripts/imgtestlib.py b/test/scripts/imgtestlib.py index 5cc17a07c5..01cbb34039 100644 --- a/test/scripts/imgtestlib.py +++ b/test/scripts/imgtestlib.py @@ -124,8 +124,9 @@ def list_images(distros=None, arches=None, images=None): # pylint: disable=too-many-arguments,too-many-positional-arguments def dl_build_cache( - destination, distro: Optional[str]=None, arch: Optional[str]=None, osbuild_ref: Optional[str]=None, - runner_distro: Optional[str]=None, manifest_id: Optional[str]=None, include_only: Optional[List[str]]=None): + destination, distro: Optional[str] = None, arch: Optional[str] = None, osbuild_ref: Optional[str] = None, + runner_distro: Optional[str] = None, manifest_id: Optional[str] = None, + include_only: Optional[List[str]] = None): """ Downloads image build cache files from the s3 bucket. @@ -309,7 +310,7 @@ def _is_bootc_manifest(manifest_data): # pylint: disable=too-many-return-statements,too-many-branches def can_boot_test(manifest_fname, manifest_data, image_type, arch, distro, blueprint): - if not image_type in CAN_BOOT_TEST.get("*", []) + CAN_BOOT_TEST.get(arch, []): + if image_type not in CAN_BOOT_TEST.get("*", []) + CAN_BOOT_TEST.get(arch, []): return False if image_type in ["image-installer", "minimal-installer"]: @@ -637,6 +638,7 @@ def skopeo_inspect_id(image_name: str, arch: str) -> str: # don't error out, just return an empty string and let the caller handle it return "" + def get_tag_for(runner): if runner.startswith("aws/"): return "terraform" @@ -645,6 +647,7 @@ def get_tag_for(runner): raise ValueError(f"Unknown runner: {runner}") + def get_ci_runner_for(arch, image_type): with open(SCHUTZFILE, encoding="utf-8") as schutzfile: data = json.load(schutzfile) From 56b234aadcc5e7ecbd747dd2ef89bb9c97b06994 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 6 Mar 2026 17:04:03 +0100 Subject: [PATCH 02/15] test: move core of build-image script to library function Make the image building scaffolding reusable. --- test/scripts/build-image | 55 ++++---------------------------------- test/scripts/imgtestlib.py | 50 ++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 50 deletions(-) diff --git a/test/scripts/build-image b/test/scripts/build-image index 6b937cc607..cc997ff491 100755 --- a/test/scripts/build-image +++ b/test/scripts/build-image @@ -1,72 +1,27 @@ #!/usr/bin/env python3 import argparse -import json import os import imgtestlib as testlib def main(): + default_arch = os.uname().machine desc = "Build image for testing with boot-image" parser = argparse.ArgumentParser(description=desc) parser.add_argument("distro", type=str, default=None, help="distro for the image to boot test") parser.add_argument("image_type", type=str, default=None, help="type of the image to boot test") parser.add_argument("config", type=str, help="config used to build the image") - parser.add_argument("--arch", default="", type=str, help="target arch of the image") + parser.add_argument("--arch", type=str, default=default_arch, + help="architecture for image (defaults to host architecture)") args = parser.parse_args() distro = args.distro image_type = args.image_type + arch = args.arch config_path = args.config - print(f"👷 Building image {distro}/{image_type} using config {config_path}") - - # print the config for logging - with open(config_path, "r", encoding="utf-8") as config_file: - config = json.load(config_file) - print(json.dumps(config, indent=2)) - config_name = config["name"] - - testlib.runcmd(["go", "build", "-o", "./bin/build", "./cmd/build"]) - - cmd = ["sudo", "-E", "./bin/build", "--output", "./build", - "--distro", distro, "--type", image_type, "--config", config_path] - arch = os.uname().machine - if args.arch: - arch = args.arch - cmd += ["--arch", args.arch] - testlib.runcmd_nc(cmd, extra_env=testlib.rng_seed_env()) - - print("✅ Build finished!!") - - # Build artifacts are owned by root. Make them world accessible. - testlib.runcmd(["sudo", "chmod", "a+rwX", "-R", "./build"]) - - build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name)) - manifest_path = os.path.join(build_dir, "manifest.json") - with open(manifest_path, "r", encoding="utf-8") as manifest_fp: - manifest_data = json.load(manifest_fp) - manifest_id = testlib.get_manifest_id(manifest_data) - - osbuild_ver, _ = testlib.runcmd(["osbuild", "--version"]) - - distro_version = testlib.get_host_distro() - osbuild_commit = testlib.get_osbuild_commit(distro_version) - if osbuild_commit is None: - osbuild_commit = "RELEASE" - - build_info = { - "distro": distro, - "arch": arch, - "image-type": image_type, - "config": config_name, - "manifest-checksum": manifest_id, - "osbuild-version": osbuild_ver.decode().strip(), - "osbuild-commit": osbuild_commit, - "commit": os.environ.get("CI_COMMIT_SHA", "N/A"), - "runner-distro": distro_version, - } - testlib.write_build_info(build_dir, build_info) + testlib.build_image(distro, arch, image_type, config_path) if __name__ == "__main__": diff --git a/test/scripts/imgtestlib.py b/test/scripts/imgtestlib.py index 01cbb34039..013f61c83a 100644 --- a/test/scripts/imgtestlib.py +++ b/test/scripts/imgtestlib.py @@ -309,6 +309,56 @@ def _is_bootc_manifest(manifest_data): # pylint: disable=too-many-return-statements,too-many-branches +def build_image(distro, arch, image_type, config_path): + with open(config_path, "r", encoding="utf-8") as config_file: + config = json.load(config_file) + + config_name = config["name"] + + print(f"👷 Building image {distro}/{image_type} using config {config_path}") + + # print the config for logging + print(json.dumps(config, indent=2)) + + runcmd(["go", "build", "-o", "./bin/build", "./cmd/build"]) + + cmd = ["sudo", "-E", "./bin/build", "--output", "./build", "--checkpoints", "build", + "--distro", distro, "--arch", arch, "--type", image_type, "--config", config_path] + runcmd_nc(cmd, extra_env=rng_seed_env()) + + print("✅ Build finished!!") + + # Build artifacts are owned by root. Make them world accessible. + runcmd(["sudo", "chmod", "a+rwX", "-R", "./build"]) + + build_dir = os.path.join("build", gen_build_name(distro, arch, image_type, config_name)) + manifest_path = os.path.join(build_dir, "manifest.json") + with open(manifest_path, "r", encoding="utf-8") as manifest_fp: + manifest_data = json.load(manifest_fp) + manifest_id = get_manifest_id(manifest_data) + + osbuild_ver, _ = runcmd(["osbuild", "--version"]) + + distro_version = get_host_distro() + osbuild_commit = get_osbuild_commit(distro_version) + if osbuild_commit is None: + osbuild_commit = "RELEASE" + + build_info = { + "distro": distro, + "arch": arch, + "image-type": image_type, + "config": config_name, + "manifest-checksum": manifest_id, + "osbuild-version": osbuild_ver.decode().strip(), + "osbuild-commit": osbuild_commit, + "commit": os.environ.get("CI_COMMIT_SHA", "N/A"), + "runner-distro": distro_version, + } + write_build_info(build_dir, build_info) + + +# pylint: disable=too-many-return-statements def can_boot_test(manifest_fname, manifest_data, image_type, arch, distro, blueprint): if image_type not in CAN_BOOT_TEST.get("*", []) + CAN_BOOT_TEST.get(arch, []): return False From 2ae6f45126a9ba033d24483928586f9ef40e3676 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Tue, 3 Mar 2026 18:33:31 +0100 Subject: [PATCH 03/15] test/imgtestlib: collapsible log sections Add functions for printing a log section start and end line.. If running in GitLab CI, use their custom collapsible sections escape sequences [1] to create collapsible build logs. [1] https://docs.gitlab.com/ci/jobs/job_logs/#custom-collapsible-sections --- test/scripts/imgtestlib.py | 42 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/test/scripts/imgtestlib.py b/test/scripts/imgtestlib.py index 013f61c83a..6ddfba0705 100644 --- a/test/scripts/imgtestlib.py +++ b/test/scripts/imgtestlib.py @@ -315,7 +315,8 @@ def build_image(distro, arch, image_type, config_path): config_name = config["name"] - print(f"👷 Building image {distro}/{image_type} using config {config_path}") + log_section_name = f"build_{distro}_{image_type}_{config_name}" + print_section_start(log_section_name, f"👷 Building image {distro}/{image_type} using config {config_path}") # print the config for logging print(json.dumps(config, indent=2)) @@ -356,6 +357,7 @@ def build_image(distro, arch, image_type, config_path): "runner-distro": distro_version, } write_build_info(build_dir, build_info) + print_section_end(log_section_name) # pylint: disable=too-many-return-statements @@ -485,7 +487,7 @@ def filter_builds(manifests, distro=None, arch=None, skip_ostree_pull=True): """ Returns a list of build requests for the manifests that have no matching config in the test build cache. """ - print(f"⚙️ Filtering {len(manifests)} build configurations") + print_section_start("filter-build-configs", f"⚙️ Filtering {len(manifests)} build configurations") dl_root_path = os.path.join(TEST_CACHE_ROOT, "s3configs", "builds") dl_path = os.path.join(dl_root_path, gen_build_info_dir_path_prefix(distro, arch)) os.makedirs(dl_path, exist_ok=True) @@ -536,6 +538,7 @@ def filter_builds(manifests, distro=None, arch=None, skip_ostree_pull=True): print("⚠️ Errors:") print("\n".join(errors)) + print_section_end("filter-build-configs") return build_requests @@ -799,3 +802,38 @@ def touch_s3(distro, arch, manifest_id, osbuild_ref=None, runner_distro=None): print(f"⌚ Updating timestamps for {s3url} ({now})") cmd = ["aws", "s3", "cp", "--recursive", "--metadata", f"touched={now}", s3url, s3url] runcmd_nc(cmd) + + +def running_in_gitlab(): + """ + Returns true if running in GitLab CI. + """ + return os.environ.get("GITLAB_CI") + + +def print_section_start(name: str, msg: str = ""): + """ + Prints a section header with a timestamp for logging output during tests. + If running in GitLab CI, it also creates a collapsible section. + + https://docs.gitlab.com/ci/jobs/job_logs/#custom-collapsible-sections + """ + now = datetime.now() + if running_in_gitlab(): + print(f"\033[0Ksection_start:{int(now.timestamp())}:{name}[collapsed=true]\r\033[0K{msg}") + return + + # custom line for non CI runs + isonow = now.isoformat() + print(f":: [{isonow}] {msg} ({name})") + + +def print_section_end(name: str): + now = datetime.now() + if running_in_gitlab(): + print(f"\033[0Ksection_end:{int(now.timestamp())}:{name}\r\033[0K") + return + + # custom line for non CI runs + isonow = now.isoformat() + print(f":: [{isonow}] Done ({name})") From a72929e31d7a320fe950b671bddb98f528e8f343 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Mon, 23 Mar 2026 18:17:54 +0100 Subject: [PATCH 04/15] test: move imgtestlib into a module directory Move imgtestlib.py to an importable module directory by the same (logical) name. --- test/scripts/imgtestlib/__init__.py | 1 + test/scripts/{ => imgtestlib}/imgtestlib.py | 2 +- test/scripts/test_imgtestlib.py | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 test/scripts/imgtestlib/__init__.py rename test/scripts/{ => imgtestlib}/imgtestlib.py (99%) diff --git a/test/scripts/imgtestlib/__init__.py b/test/scripts/imgtestlib/__init__.py new file mode 100644 index 0000000000..2d3a0ae3f6 --- /dev/null +++ b/test/scripts/imgtestlib/__init__.py @@ -0,0 +1 @@ +from .imgtestlib import * diff --git a/test/scripts/imgtestlib.py b/test/scripts/imgtestlib/imgtestlib.py similarity index 99% rename from test/scripts/imgtestlib.py rename to test/scripts/imgtestlib/imgtestlib.py index 6ddfba0705..436c305c61 100644 --- a/test/scripts/imgtestlib.py +++ b/test/scripts/imgtestlib/imgtestlib.py @@ -18,7 +18,7 @@ REGISTRY = "registry.gitlab.com/redhat/services/products/image-builder/ci/images" # Path to the Schutzfile relative to the root of the repository -SCHUTZFILE = str(pathlib.Path(__file__).resolve().parents[2] / "Schutzfile") +SCHUTZFILE = str(pathlib.Path(__file__).resolve().parents[3] / "Schutzfile") OS_RELEASE_FILE = "/etc/os-release" # image types that can be boot tested diff --git a/test/scripts/test_imgtestlib.py b/test/scripts/test_imgtestlib.py index efd1858f58..61232a182a 100644 --- a/test/scripts/test_imgtestlib.py +++ b/test/scripts/test_imgtestlib.py @@ -114,8 +114,8 @@ def test_read_seed(): ), )) def test_gen_build_info_dir_path_prefix(kwargs, expected): - with patch("imgtestlib.get_host_distro", return_value="fedora-999"), \ - patch("imgtestlib.get_osbuild_commit", return_value="abcdef123456"): + with patch("imgtestlib.imgtestlib.get_host_distro", return_value="fedora-999"), \ + patch("imgtestlib.imgtestlib.get_osbuild_commit", return_value="abcdef123456"): assert testlib.gen_build_info_dir_path_prefix(**kwargs) == expected @@ -201,8 +201,8 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): ), )) def test_gen_build_info_s3_dir_path(kwargs, expected): - with patch("imgtestlib.get_host_distro", return_value="fedora-999"), \ - patch("imgtestlib.get_osbuild_commit", return_value="abcdef123456"): + with patch("imgtestlib.imgtestlib.get_host_distro", return_value="fedora-999"), \ + patch("imgtestlib.imgtestlib.get_osbuild_commit", return_value="abcdef123456"): assert testlib.gen_build_info_s3_dir_path(**kwargs) == expected From 65682dcaf380b6237b6b2e5c1efa9587ff1435be Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Mon, 23 Mar 2026 19:00:52 +0100 Subject: [PATCH 05/15] test/imgtestlib: break up imgtestlib into multiple submodules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The module got too big for one file. This commit only moves code around and adjusts imports. Functions are grouped into submodules by logical functionality. Groups (submodule boundaries) were chosen to avoid circular imports. The .core module could still use some splitting or tidying up, but the current state is already an improvement. Importing everything into __init__.py might be unnecessary—we only really need to import the functions that are used externally—but it's not very important to reduce the API surface of this internal testing library. --- test/scripts/boot-image | 568 +---------------- test/scripts/imgtestlib/__init__.py | 8 +- test/scripts/imgtestlib/boot.py | 585 ++++++++++++++++++ test/scripts/imgtestlib/build.py | 92 +++ test/scripts/imgtestlib/cache.py | 112 ++++ test/scripts/imgtestlib/core.py | 371 ++++++++++++ test/scripts/imgtestlib/gitlab.py | 37 ++ test/scripts/imgtestlib/imgtestlib.py | 839 -------------------------- test/scripts/imgtestlib/run.py | 33 + test/scripts/imgtestlib/testenv.py | 129 ++++ test/scripts/test_imgtestlib.py | 34 +- 11 files changed, 1393 insertions(+), 1415 deletions(-) create mode 100644 test/scripts/imgtestlib/boot.py create mode 100644 test/scripts/imgtestlib/build.py create mode 100644 test/scripts/imgtestlib/cache.py create mode 100644 test/scripts/imgtestlib/core.py create mode 100644 test/scripts/imgtestlib/gitlab.py delete mode 100644 test/scripts/imgtestlib/imgtestlib.py create mode 100644 test/scripts/imgtestlib/run.py create mode 100644 test/scripts/imgtestlib/testenv.py diff --git a/test/scripts/boot-image b/test/scripts/boot-image index a9d31fdb89..05130b860d 100755 --- a/test/scripts/boot-image +++ b/test/scripts/boot-image @@ -1,560 +1,11 @@ #!/usr/bin/env python3 -# pylint: disable=line-too-long,too-many-arguments,too-many-positional-arguments import argparse -import contextlib import json import os import pathlib -import random -import shutil -import signal -import string -import subprocess -import textwrap -import uuid -from tempfile import TemporaryDirectory -from typing import Dict, Optional import imgtestlib as testlib -from vmtest.util import get_free_port -from vmtest.vm import QEMU - - -BASE_TEST_EXEC = "check-host-config-" # + arch -WSL_TEST_SCRIPT = "test/scripts/wsl-entrypoint.bat" -# We need up to 15 minutes for full Anaconda ISO installations -# But in some cases the CI systems are even slower, so use 1800s -ISO_BOOT_TIMEOUT = 1800 - - -def ensure_env_vars(env_vars: Dict[str, Optional[str]]): - missing = [name for name, value in env_vars.items() if value is None or value == ""] - if missing: - raise RuntimeError(f"Missing or empty environment variables: {missing}") - - -def get_aws_config(): - env_vars = { - "key_id": os.environ.get("AWS_ACCESS_KEY_ID"), - "secret_key": os.environ.get("AWS_SECRET_ACCESS_KEY"), - "bucket": os.environ.get("AWS_BUCKET"), - "region": os.environ.get("AWS_REGION") - } - ensure_env_vars(env_vars) - return env_vars - - -def get_azure_config(): - env_vars = { - "subscription": os.environ.get("AZURE_SUBSCRIPTION"), - "tenant": os.environ.get("AZURE_TENANT"), - "client_id": os.environ.get("AZURE_CLIENT_ID"), - "client_secret": os.environ.get("AZURE_CLIENT_SECRET"), - "resource_group": os.environ.get("AZURE_RESOURCE_GROUP"), - } - ensure_env_vars(env_vars) - return env_vars - - -def get_wsl_config(): - azure_config = get_azure_config() - env_vars = { - "windows_snapshot": os.environ.get("AZURE_WINDOWS_SNAPSHOT"), - "windows_ssh_privkey": os.environ.get("AZURE_WINDOWS_SSH_PRIVKEY"), - } - ensure_env_vars(env_vars) - env_vars.update(azure_config) - return env_vars - - -@contextlib.contextmanager -def create_ssh_key(privkey_file = None, key_type = None): - with TemporaryDirectory() as tmpdir: - keypath = os.path.join(tmpdir, "testkey") - ci_priv_key = os.environ.get("CI_PRIV_SSH_KEY") - if privkey_file is not None: - shutil.copyfile(privkey_file, keypath) - os.chmod(keypath, 0o600) - - cmd = ["ssh-keygen", "-y", "-f", keypath] - out, _ = testlib.runcmd(cmd) - pubkey = out.decode() - with open(keypath + ".pub", "w", encoding="utf-8") as pubkeyfile: - pubkeyfile.write(pubkey) - elif not key_type and ci_priv_key: - # running in CI: use key from env - with open(keypath, "w", encoding="utf-8") as keyfile: - keyfile.write(ci_priv_key + "\n") - os.chmod(keypath, 0o600) - - # get public key from priv key and write it out - cmd = ["ssh-keygen", "-y", "-f", keypath] - out, _ = testlib.runcmd(cmd) - pubkey = out.decode() - with open(keypath + ".pub", "w", encoding="utf-8") as pubkeyfile: - pubkeyfile.write(pubkey) - elif key_type == "rsa": - cmd = ["ssh-keygen", "-t", "rsa", "-b", "2048", "-m", "pem", "-N", "", "-f", keypath] - testlib.runcmd_nc(cmd) - else: - # create an ssh key pair with empty password - cmd = ["ssh-keygen", "-t", "ecdsa", "-b", "256", "-m", "pem", "-N", "", "-f", keypath] - testlib.runcmd_nc(cmd) - - yield keypath, keypath + ".pub" - - -@contextlib.contextmanager -def ensure_uncompressed(filepath): - """ - If the file at the given path is compressed, decompress it and return the new file path. - """ - base, ext = os.path.splitext(filepath) - if ext == ".xz": - print(f"Uncompressing {filepath}") - # needs to run as root to set perms and ownership on uncompressed file - testlib.runcmd_nc(["sudo", "unxz", "--verbose", "--keep", filepath]) - yield base - # cleanup when done so the uncompressed file doesn't get uploaded to the build cache - os.unlink(base) - - else: - # we only do xz for now so it must be raw: return as is and hope for the best - yield filepath - - -@contextlib.contextmanager -def make_cloud_init_iso(pubkey_path) -> pathlib.Path: - ssh_key = pathlib.Path(pubkey_path).read_text(encoding="utf8").strip() - with TemporaryDirectory() as tmpdir: - user_data = pathlib.Path(tmpdir) / "user-data.yaml" - user_data_content = textwrap.dedent(f"""\ - #cloud-config - users: - - name: root - ssh_authorized_keys: - - {ssh_key} - - name: osbuild - groups: [wheel] - sudo: ALL=(ALL) NOPASSWD:ALL - ssh_authorized_keys: - - {ssh_key} - """) - user_data.write_text(user_data_content) - meta_data = pathlib.Path(tmpdir) / "meta-data" - meta_data.write_text('{"instance-id": "i-1234567890abcdef0"}') - iso_path = pathlib.Path(tmpdir) / "cloud-init.iso" - subprocess.check_call( - ["cloud-localds", os.fspath(iso_path), user_data.name, meta_data.name], - cwd=tmpdir, - ) - yield iso_path - - -def arch_to_goarch(arch): - """ - Convert architecture string to GOARCH format. - """ - mapping = { - "x86_64": "amd64", - "aarch64": "arm64", - } - goarch = mapping.get(arch.lower()) - if goarch is None: - return arch - return goarch - - -def make_check_host_config(arch): - goarch = arch_to_goarch(arch) - # build without CGO so no dependencies are needed - cmd = ["go", "build", "-o", "check-host-config-" + arch, - "./cmd/check-host-config"] - tags = [ - "containers_image_openpgp", - "exclude_graphdriver_btrfs", - "exclude_graphdriver_devicemapper", - "exclude_graphdriver_overlay", - ] - testlib.runcmd_nc( - cmd, - extra_env={ - "GOARCH": goarch, - "CGO_ENABLED": "0", - "GOFLAGS": "-tags=" + ",".join(tags), - }, - ) - - -class CannotRunQemuTest(Exception): - def __init__(self, skip_reason): - super().__init__(skip_reason) - self.skip_reason = skip_reason - - -class MissingBootImplementation(Exception): - def __init__(self, skip_reason): - super().__init__(skip_reason) - self.skip_reason = skip_reason - - -def qemu_cmd_scp_and_run(vm, cmd, privkey_path): - # This is similar to what the other runners are doing but - # it would be nice to find a better way, e.g. create a - # bundle or compsoe a single script with the config - # build-in/appended - for arg in cmd: - if os.path.exists(arg): - vm.scp(arg, "/tmp/", user="osbuild", keyfile=privkey_path) - vmcmd = ["/tmp/" + os.path.basename(arg) for arg in cmd] - return vm.run(vmcmd, user="osbuild", keyfile=privkey_path) - - -def boot_qemu(arch, image_path, config_file, keep_booted=False): - cmd = [BASE_TEST_EXEC+arch, config_file] - make_check_host_config(arch) - with contextlib.ExitStack() as cm: - uncompressed_image_path = cm.enter_context(ensure_uncompressed(image_path)) - (privkey_path, pubkey_path) = cm.enter_context(create_ssh_key()) - cloud_init_iso = cm.enter_context(make_cloud_init_iso(pubkey_path)) - with QEMU(uncompressed_image_path, arch=arch, cdrom=cloud_init_iso) as vm: - try: - qemu_cmd_scp_and_run(vm, cmd, privkey_path) - finally: - if keep_booted: - print("***********************************") - print(f"keeping the image {image_path} booted as requested, press enter or ctrl-c to stop") - print("to connect run:") - print( - f"ssh -i {privkey_path} -p {vm.ssh_port} -o UserKnownHostsFile=/dev/null " - "-o StrictHostKeyChecking=no osbuild@localhost") - signal.pause() - - -def boot_qemu_iso_no_unattended_support(distro, arch, image_type, installer_iso_path, config_file, iso_embedded_ks_path=None): - """ - Boot an ISO that has no "unattended" support in its blueprint. - Manually create a custom kickstart file and modify the ISO to use it. - """ - # If an embedded kickstart was provided, prepend its content to preserve original directives - # (unless overridden by the unattended automation directives added below). - custom_ks_content = None - if iso_embedded_ks_path: - with open(iso_embedded_ks_path, "r", encoding="utf-8") as fp: - custom_ks_content = fp.read() - print(f"ISO embedded kickstart content: \n{custom_ks_content}\n\n") - - rootpw = "".join(random.choices(string.ascii_uppercase + string.digits, k=18)) - rhsm = "" - rhsm_unregister = "" - - # NOTE: we do not need to register the bootc image, since all content comes from the container - if distro.startswith("rhel") and not image_type.startswith("bootc-"): - org_id = os.getenv("SUBSCRIPTION_ORG") - activation_key = os.getenv("SUBSCRIPTION_ACTIVATION_KEY") - if not org_id or not activation_key: - raise CannotRunQemuTest("rhel unattended tests need SUBSCRIPTION_ORG and SUBSCRIPTION_ACTIVATION_KEY env") - rhsm = f'rhsm --organization="{org_id}" --activation-key="{activation_key}"' - rhsm_unregister = textwrap.dedent("""\ - # ensure we unregister after the install again, no need to keep the system registered - # and show up in the inventory - subscription-manager unregister - """) - - with contextlib.ExitStack() as cm: - tmpdir = cm.enter_context(TemporaryDirectory(dir="/var/tmp")) - (privkey_path, pubkey_path) = cm.enter_context(create_ssh_key()) - pubkey = pathlib.Path(pubkey_path).read_text("utf8").strip() - unattended_ks = pathlib.Path(tmpdir) / "ks.cfg" - - ks_content = "" - if custom_ks_content: - ks_content += "# Content of the original ks embedded in the ISO\n" - ks_content += custom_ks_content + "\n\n" - - ks_content += textwrap.dedent(f"""\ - # Unattended automation directives generated by boot_qemu_iso_no_unattended_support() - text --non-interactive - zerombr - clearpart --all --initlabel - autopart --type=plain - network --activate --onboot=on - reboot --eject - user --name=osbuild --group=wheel --shell=/bin/bash - sshkey --username=osbuild "{pubkey}" - rootpw {rootpw} - {rhsm} - eula --agree - # better debug for the sshd failure - bootloader --append="console=ttyS0 systemd.journald.forward_to_console=1" - %post - # workaround for centos-10 as it fails to start here and that causes issue - # with our "check-host-config.sh" that expects a non-degraded boot - systemctl mask mcelog.service || true - echo "osbuild ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/osbuild - chmod 0440 /etc/sudoers.d/osbuild - {rhsm_unregister} - %end - """) - unattended_ks.write_text(ks_content) - - new_installer_iso_path = pathlib.Path(tmpdir) / os.path.basename(installer_iso_path) - subprocess.check_call( - ["sudo", "mkksiso", - # Note that we could add: - # systemd.journald.forward_to_console=1 - # here as well but it produces extrem amounts of logs - # that exceeds the gitlab limit - "-c", "console=ttyS0", - "--ks", os.fspath(unattended_ks), - os.fspath(installer_iso_path), new_installer_iso_path]) - return _boot_qemu_iso(arch, new_installer_iso_path, config_file, privkey_path) - - -def boot_qemu_iso(arch, installer_iso_path, config_file): - # We can only test the unattended-iso as the other configs require - # interactive setup of the installer which we do not support in this - # test-runner. - with contextlib.ExitStack() as cm: - # The "unattended-iso" has the CI ssh key, so if we are running - # in CI we can actually do a real image test. Sadly not locally - # because we have no way to log into the installed disk in this - # case, the CI ssh key is secret. - privkey_path = None - if os.environ.get("CI_PRIV_SSH_KEY"): - (privkey_path, _) = cm.enter_context(create_ssh_key()) - return _boot_qemu_iso(arch, installer_iso_path, config_file, privkey_path) - - -def _boot_qemu_iso(arch, installer_iso_path, config_file, privkey_path): - cmd = [BASE_TEST_EXEC+arch, config_file] - make_check_host_config(arch) - # we should pass console=ttyS0 to instaler os that we see install - # progress on the serial console, in the meantime for interactive use - # one cans set the OSBUILD_TEST_QEMU_GUI=1 environment - with contextlib.ExitStack() as cm: - # we need /var/tmp here as /tmp might be on a (small) tmpfs - tmpdir = cm.enter_context(TemporaryDirectory(dir="/var/tmp")) - # create an (empty) target disk, truncate ensures its sparse - # so it will not take up real disk space until things are - # written to it - test_disk_path = pathlib.Path(tmpdir) / "disk.img" - with open(test_disk_path, "w", encoding="utf8") as fp: - fp.truncate(20_000_000_000) - # boot from installer to install to test disk, anaconda will - # reboot automatically for the unattended-iso config. - with QEMU(test_disk_path, cdrom=installer_iso_path) as vm: - vm.start(wait_event="qmp:RESET", snapshot=False, use_ovmf=True, timeout_sec=ISO_BOOT_TIMEOUT) - vm.force_stop() - # now boot test disk and wait for ssh to come up as a minimal boot test - with QEMU(test_disk_path, arch=arch) as vm: - vm.start(use_ovmf=True) - vm.wait_ssh_ready() - if privkey_path: - qemu_cmd_scp_and_run(vm, cmd, privkey_path) - - -def boot_qemu_pxe(arch, pxe_tar_path): - with contextlib.ExitStack() as cm: - # unpack the tar and create a combined image - tmpdir = cm.enter_context(TemporaryDirectory(dir="/var/tmp")) - subprocess.check_call( - ["tar", "-C", tmpdir, "-x", "-f", pxe_tar_path]) - subprocess.check_call( - "echo rootfs.img | cpio -H newc --quiet -L -o > rootfs.cpio", shell=True, cwd=tmpdir) - subprocess.check_call( - "cat initrd.img rootfs.cpio > combined.img", shell=True, cwd=tmpdir) - - # Start an HTTP server to serve the rootfs.img and terminate it after the test. - # Explicitly terminate the HTTP server to avoid blocking on wait(), this cannot - # be done with a context manager for subprocesses. - http_port = get_free_port() - http_server = subprocess.Popen( # pylint: disable=consider-using-with - ["python3", "-m", "http.server", f"{http_port}"], - cwd=tmpdir, - # prevent blocking output - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL - ) - try: - # test disk is unused for live OS - test_disk_path = pathlib.Path(tmpdir) / "disk.img" - with open(test_disk_path, "w", encoding="utf-8") as fp: - fp.truncate(0) - - # test both the combined and HTTP rootfs variants - for use_ovmf in [False, True]: - for root_arg, initrd_file in [ - ("live:/rootfs.img", "combined.img"), - (f"live:http://10.0.2.2:{http_port}/rootfs.img", "initrd.img") - ]: - append_arg = ( - f"rd.live.image root={root_arg} console=ttyS0 " - f"systemd.debug-shell=ttyS0 " - f"systemd.mask=serial-getty@ttyS0.service " - f"systemd.unit=reboot.target" - ) - extra_args = [ - "-kernel", str(pathlib.Path(tmpdir) / "vmlinuz"), - "-initrd", str(pathlib.Path(tmpdir) / initrd_file), - "-append", append_arg - ] - - with QEMU(test_disk_path, memory="3000", arch=arch, extra_args=extra_args) as vm: - # Wait for QMP RESET event instead of SSH since PXE images don't have SSH. - # The systemd.unit=reboot.target will cause a reboot, triggering the RESET event. - vm.start(wait_event="qmp:RESET", snapshot=False, use_ovmf=use_ovmf) - # There is really very little in the rootfs.img (i.e. no ssh, cloud-init - # or other things that open ports or dnf) and we can only control it via the - # kernel commandline. So via the "systemd.unit=reboot.target" kernel commandline - # above we boot and then force a reboot right away as our test. This is not great - # but the best we can do right now. Other options: - # 1. have a blueprint with sshd-server so that we can check for ssh port - # 2. modify vm.py to be able to talk directly to the serial console - # and then run commands directly via that - vm.force_stop() - finally: - http_server.terminate() - http_server.wait() - - -def cmd_boot_aws(arch, image_name, privkey, pubkey, image_path, script_cmd): - make_check_host_config(arch) - aws_config = get_aws_config() - cmd = ["go", "run", "./cmd/boot-aws", "run", - "--access-key-id", aws_config["key_id"], - "--secret-access-key", aws_config["secret_key"], - "--region", aws_config["region"], - "--bucket", aws_config["bucket"], - "--arch", arch, - "--ami-name", image_name, - "--s3-key", f"images/boot/{image_name}", - "--username", "osbuild", - "--ssh-privkey", privkey, - "--ssh-pubkey", pubkey, - image_path, *script_cmd] - testlib.runcmd_nc(cmd) - - -def boot_ami(distro, arch, image_type, image_path, config): - cmd = [BASE_TEST_EXEC+arch, config] - make_check_host_config(arch) - with ensure_uncompressed(image_path) as raw_image_path: - with create_ssh_key() as (privkey, pubkey): - image_name = f"image-boot-test-{distro}-{arch}-{image_type}-" + str(uuid.uuid4()) - cmd_boot_aws(arch, image_name, privkey, pubkey, raw_image_path, cmd) - - -def boot_container(distro, arch, image_type, image_path, manifest_id, host_config): - """ - Use bootc-image-builder to build an AMI and boot it. - """ - # push container to registry so we can build it with BIB - # remove when BIB can pull from containers-storage: https://github.com/osbuild/bootc-image-builder/pull/120 - container_name = f"iot-bootable-container:{distro}-{arch}-{manifest_id}" - cmd = ["./tools/ci/push-container.sh", image_path, container_name] - testlib.runcmd_nc(cmd) - container_ref = f"{testlib.REGISTRY}/{container_name}" - - with TemporaryDirectory() as tmpdir: - with create_ssh_key() as (privkey_file, pubkey_file): - with open(pubkey_file, encoding="utf-8") as pubkey_fp: - pubkey = pubkey_fp.read() - - # write a config to create a user - config_file = os.path.join(tmpdir, "config.json") - with open(config_file, "w", encoding="utf-8") as cfg_fp: - config = { - "blueprint": { - "customizations": { - "user": [ - { - "name": "osbuild", - "key": pubkey, - "groups": [ - "wheel" - ] - } - ] - } - } - } - json.dump(config, cfg_fp) - - # build an AMI - cmd = ["sudo", "podman", "run", - "--rm", "-it", - "--privileged", - "--pull=newer", - "--security-opt", "label=type:unconfined_t", - "-v", f"{tmpdir}:/output", - "-v", f"{config_file}:/config.json", - testlib.get_bib_ref(), - "--type=ami", - "--config=/config.json", - container_ref] - testlib.runcmd_nc(cmd) - - # boot it - image_name = f"image-boot-test-{distro}-{arch}-{image_type}-" + str(uuid.uuid4()) - - # Build artifacts are owned by root. Make them world accessible. - testlib.runcmd(["sudo", "chmod", "a+rwX", "-R", tmpdir]) - raw_image_path = f"{tmpdir}/image/disk.raw" - cmd_boot_aws(arch, image_name, privkey_file, pubkey_file, raw_image_path, [BASE_TEST_EXEC+arch, host_config]) - - -def boot_vhd(distro, arch, image_path, config): - cmd = [BASE_TEST_EXEC+arch, config] - make_check_host_config(arch) - with ensure_uncompressed(image_path) as raw_image_path: - with create_ssh_key(key_type="rsa") as (privkey, pubkey): - # a lot of resources have <=64 character naming constraint - name = f"{distro}-" + str(uuid.uuid4()) - - az_config = get_azure_config() - cmd = ["go", "run", "./cmd/boot-azure", "run", - "--subscription", az_config["subscription"], - "--tenant", az_config["tenant"], - "--client-id", az_config["client_id"], - "--client-secret", az_config["client_secret"], - "--resource-group", az_config["resource_group"], - "--username", "osbuild", - "--ssh-privkey", privkey, - "--ssh-pubkey", pubkey, - "--vm-name", name, - "--arch", arch, - "--image-name", name, - raw_image_path, *cmd] - testlib.runcmd_nc(cmd) - - -def boot_wsl(distro, arch, image_path, config): - with ensure_uncompressed(image_path) as raw_image_path: - cmd = [WSL_TEST_SCRIPT, raw_image_path, BASE_TEST_EXEC+arch, config] - make_check_host_config(arch) - az_config = get_wsl_config() - with create_ssh_key(privkey_file = az_config["windows_ssh_privkey"]) as (privkey, pubkey): - # a lot of resources have <=64 character naming constraint - name = f"{distro}-" + str(uuid.uuid4()) - - cmd = ["go", "run", "./cmd/boot-azure", "run", - "--subscription", az_config["subscription"], - "--tenant", az_config["tenant"], - "--client-id", az_config["client_id"], - "--client-secret", az_config["client_secret"], - "--resource-group", az_config["resource_group"], - "--snapshot", az_config["windows_snapshot"], - "--username", "azureuser", - "--ssh-privkey", privkey, - "--ssh-pubkey", pubkey, - "--vm-name", name, - "--arch", arch, - "--size", "Standard_D2as_v5", - raw_image_path, *cmd] - testlib.runcmd_nc(cmd) - # pylint: disable=too-many-branches def main(): @@ -599,30 +50,29 @@ def main(): # Not all qcow2 types can be boot-tested, for example `server-qcow2` uses # initial-setup and this blocks the boot. case "qcow2" | "generic-qcow2" | "cloud-qcow2": - boot_qemu(arch, image_path, build_config_path, keep_booted=args.keep_booted) + testlib.boot_qemu(arch, image_path, build_config_path, keep_booted=args.keep_booted) case "image-installer" | "minimal-installer": - boot_qemu_iso(arch, image_path, build_config_path) + testlib.boot_qemu_iso(arch, image_path, build_config_path) case "network-installer" | "everything-network-installer" | "bootc-generic-iso": - boot_qemu_iso_no_unattended_support(distro, arch, image_type, image_path, build_config_path, iso_embedded_ks_path) + testlib.boot_qemu_iso_no_unattended_support(arch, image_path, build_config_path) case "pxe-tar-xz": - boot_qemu_pxe(arch, image_path) + testlib.boot_qemu_pxe(arch, image_path) case "ami" | "ec2" | "ec2-ha" | "ec2-sap" | "edge-ami" | "cloud-ec2": - boot_ami(distro, arch, image_type, image_path, build_config_path) + testlib.boot_ami(distro, arch, image_type, image_path, build_config_path) case "vhd": - boot_vhd(distro, arch, image_path, build_config_path) + testlib.boot_vhd(distro, arch, image_path, build_config_path) case "iot-bootable-container": manifest_id = build_info["manifest-checksum"] - boot_container(distro, arch, image_type, image_path, manifest_id, build_config_path) + testlib.boot_container(distro, arch, image_type, image_path, manifest_id, build_config_path) bib_ref = testlib.get_bib_ref() bib_image_id = testlib.skopeo_inspect_id(f"docker://{bib_ref}", testlib.host_container_arch()) case "wsl" | "generic-wsl": if distro == "fedora-41": print(f"{distro} {image_type} boot tests are not supported, fails on wsl import") return - boot_wsl(distro, arch, image_path, build_config_path) + testlib.boot_wsl(distro, arch, image_path, build_config_path) case _: - # skip - raise MissingBootImplementation(f"{arch} {image_type} is missing a boot implementation.") + raise testlib.MissingBootImplementation(f"{arch} {image_type} is missing a boot implementation.") print("✅ Marking boot successful") # amend build info with boot success diff --git a/test/scripts/imgtestlib/__init__.py b/test/scripts/imgtestlib/__init__.py index 2d3a0ae3f6..fe1d86bfbb 100644 --- a/test/scripts/imgtestlib/__init__.py +++ b/test/scripts/imgtestlib/__init__.py @@ -1 +1,7 @@ -from .imgtestlib import * +from .boot import * +from .build import * +from .cache import * +from .core import * +from .gitlab import * +from .run import * +from .testenv import * diff --git a/test/scripts/imgtestlib/boot.py b/test/scripts/imgtestlib/boot.py new file mode 100644 index 0000000000..e1c987a771 --- /dev/null +++ b/test/scripts/imgtestlib/boot.py @@ -0,0 +1,585 @@ +import contextlib +import json +import os +import pathlib +import random +import shutil +import signal +import string +import subprocess +import textwrap +import uuid +from tempfile import TemporaryDirectory +from typing import Generator + +from vmtest.util import get_free_port +from vmtest.vm import QEMU + +from .run import runcmd, runcmd_nc +from .testenv import get_bib_ref + +BASE_TEST_EXEC = "check-host-config-" # + arch +WSL_TEST_SCRIPT = "test/scripts/wsl-entrypoint.bat" +# We need up to 15 minutes for full Anaconda ISO installations +# But in some cases the CI systems are even slower, so use 1800s +ISO_BOOT_TIMEOUT = 1800 + +REGISTRY = "registry.gitlab.com/redhat/services/products/image-builder/ci/images" + +# image types that can be boot tested +# Keep in sync with test/scripts/boot-image which has the same checks again +CAN_BOOT_TEST = { + "*": [ + "ami", + "ec2", + "ec2-ha", + "ec2-sap", + "edge-ami", + "iot-bootable-container", + "vhd", + "cloud-ec2", + ], + "x86_64": [ + "image-installer", "minimal-installer", "network-installer", + "qcow2", "generic-qcow2", "cloud-qcow2", + "wsl", "generic-wsl", + ] +} + + +def get_aws_config(): + return { + "key_id": os.environ.get("AWS_ACCESS_KEY_ID"), + "secret_key": os.environ.get("AWS_SECRET_ACCESS_KEY"), + "bucket": os.environ.get("AWS_BUCKET"), + "region": os.environ.get("AWS_REGION") + } + + +def get_azure_config(): + return { + "subscription": os.environ.get("AZURE_SUBSCRIPTION"), + "tenant": os.environ.get("AZURE_TENANT"), + "client_id": os.environ.get("AZURE_CLIENT_ID"), + "client_secret": os.environ.get("AZURE_CLIENT_SECRET"), + "resource_group": os.environ.get("AZURE_RESOURCE_GROUP"), + "windows_snapshot": os.environ.get("AZURE_WINDOWS_SNAPSHOT"), + "windows_ssh_privkey": os.environ.get("AZURE_WINDOWS_SSH_PRIVKEY"), + } + + +@contextlib.contextmanager +def create_ssh_key(privkey_file=None, key_type=None): + with TemporaryDirectory() as tmpdir: + keypath = os.path.join(tmpdir, "testkey") + ci_priv_key = os.environ.get("CI_PRIV_SSH_KEY") + if privkey_file is not None: + shutil.copyfile(privkey_file, keypath) + os.chmod(keypath, 0o600) + + cmd = ["ssh-keygen", "-y", "-f", keypath] + out, _ = runcmd(cmd) + pubkey = out.decode() + with open(keypath + ".pub", "w", encoding="utf-8") as pubkeyfile: + pubkeyfile.write(pubkey) + elif not key_type and ci_priv_key: + # running in CI: use key from env + with open(keypath, "w", encoding="utf-8") as keyfile: + keyfile.write(ci_priv_key + "\n") + os.chmod(keypath, 0o600) + + # get public key from priv key and write it out + cmd = ["ssh-keygen", "-y", "-f", keypath] + out, _ = runcmd(cmd) + pubkey = out.decode() + with open(keypath + ".pub", "w", encoding="utf-8") as pubkeyfile: + pubkeyfile.write(pubkey) + elif key_type == "rsa": + cmd = ["ssh-keygen", "-t", "rsa", "-b", "2048", "-m", "pem", "-N", "", "-f", keypath] + runcmd_nc(cmd) + else: + # create an ssh key pair with empty password + cmd = ["ssh-keygen", "-t", "ecdsa", "-b", "256", "-m", "pem", "-N", "", "-f", keypath] + runcmd_nc(cmd) + + yield keypath, keypath + ".pub" + + +@contextlib.contextmanager +def ensure_uncompressed(filepath): + """ + If the file at the given path is compressed, decompress it and return the new file path. + """ + base, ext = os.path.splitext(filepath) + if ext == ".xz": + print(f"Uncompressing {filepath}") + # needs to run as root to set perms and ownership on uncompressed file + runcmd_nc(["sudo", "unxz", "--verbose", "--keep", filepath]) + yield base + # cleanup when done so the uncompressed file doesn't get uploaded to the build cache + os.unlink(base) + + else: + # we only do xz for now so it must be raw: return as is and hope for the best + yield filepath + + +@contextlib.contextmanager +def make_cloud_init_iso(pubkey_path) -> Generator: + ssh_key = pathlib.Path(pubkey_path).read_text(encoding="utf8").strip() + with TemporaryDirectory() as tmpdir: + user_data = pathlib.Path(tmpdir) / "user-data.yaml" + user_data_content = textwrap.dedent(f"""\ + #cloud-config + users: + - name: root + ssh_authorized_keys: + - {ssh_key} + - name: osbuild + groups: [wheel] + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - {ssh_key} + """) + user_data.write_text(user_data_content) + meta_data = pathlib.Path(tmpdir) / "meta-data" + meta_data.write_text('{"instance-id": "i-1234567890abcdef0"}') + iso_path = pathlib.Path(tmpdir) / "cloud-init.iso" + subprocess.check_call( + ["cloud-localds", os.fspath(iso_path), user_data.name, meta_data.name], + cwd=tmpdir, + ) + yield iso_path + + +def arch_to_goarch(arch): + """ + Convert architecture string to GOARCH format. + """ + mapping = { + "x86_64": "amd64", + "aarch64": "arm64", + } + goarch = mapping.get(arch.lower()) + if goarch is None: + return arch + return goarch + + +def make_check_host_config(arch): + goarch = arch_to_goarch(arch) + # build without CGO so no dependencies are needed + cmd = ["go", "build", "-o", "check-host-config-" + arch, + "./cmd/check-host-config"] + tags = [ + "containers_image_openpgp", + "exclude_graphdriver_btrfs", + "exclude_graphdriver_devicemapper", + "exclude_graphdriver_overlay", + ] + runcmd_nc( + cmd, + extra_env={ + "GOARCH": goarch, + "CGO_ENABLED": "0", + "GOFLAGS": "-tags=" + ",".join(tags), + }, + ) + + +class CannotRunQemuTest(Exception): + def __init__(self, skip_reason): + super().__init__(skip_reason) + self.skip_reason = skip_reason + + +class MissingBootImplementation(Exception): + def __init__(self, skip_reason): + super().__init__(skip_reason) + self.skip_reason = skip_reason + + +def qemu_cmd_scp_and_run(vm, cmd, privkey_path): + # This is similar to what the other runners are doing but + # it would be nice to find a better way, e.g. create a + # bundle or compsoe a single script with the config + # build-in/appended + for arg in cmd: + if os.path.exists(arg): + vm.scp(arg, "/tmp/", user="osbuild", keyfile=privkey_path) + vmcmd = ["/tmp/" + os.path.basename(arg) for arg in cmd] + return vm.run(vmcmd, user="osbuild", keyfile=privkey_path) + + +def boot_qemu(arch, image_path, config_file, keep_booted=False): + cmd = [BASE_TEST_EXEC+arch, config_file] + make_check_host_config(arch) + with contextlib.ExitStack() as cm: + uncompressed_image_path = cm.enter_context(ensure_uncompressed(image_path)) + (privkey_path, pubkey_path) = cm.enter_context(create_ssh_key()) + cloud_init_iso = cm.enter_context(make_cloud_init_iso(pubkey_path)) + with QEMU(uncompressed_image_path, arch=arch, cdrom=cloud_init_iso) as vm: + try: + qemu_cmd_scp_and_run(vm, cmd, privkey_path) + finally: + if keep_booted: + print("***********************************") + print(f"keeping the image {image_path} booted as requested, press enter or ctrl-c to stop") + print("to connect run:") + print( + f"ssh -i {privkey_path} -p {vm.ssh_port} -o UserKnownHostsFile=/dev/null " + "-o StrictHostKeyChecking=no osbuild@localhost") + signal.pause() + + +def boot_qemu_iso_no_unattended_support(arch, installer_iso_path, config_file): + # this is for ISOs that have no "unattneded" support in their blueprint, + # manually create one and modify the ISO + rootpw = "".join( + random.choices(string.ascii_uppercase + string.digits, k=18)) + rhsm = "" + rhsm_unregister = "" + # this is too crude, use "distro" from info file + if "rhel" in installer_iso_path: + org_id = os.getenv("SUBSCRIPTION_ORG") + activation_key = os.getenv("SUBSCRIPTION_ACTIVATION_KEY") + if not org_id or not activation_key: + raise CannotRunQemuTest("rhel unattended tests need SUBSCRIPTION_ORG and SUBSCRIPTION_ACTIVATION_KEY env") + rhsm = f'rhsm --organization="{org_id}" --activation-key="{activation_key}"' + rhsm_unregister = textwrap.dedent("""\ + # ensure we unregister after the install again, no need to keep the system registered + # and show up in the inventory + subscription-manager unregister + """) + with contextlib.ExitStack() as cm: + tmpdir = cm.enter_context(TemporaryDirectory(dir="/var/tmp")) + (privkey_path, pubkey_path) = cm.enter_context(create_ssh_key()) + pubkey = pathlib.Path(pubkey_path).read_text("utf8").strip() + unattended_ks = pathlib.Path(tmpdir) / "ks.cfg" + unattended_ks.write_text(textwrap.dedent(f"""\ + text --non-interactive + zerombr + clearpart --all --initlabel + autopart --type=plain + network --activate --onboot=on + reboot --eject + user --name=osbuild --group=wheel --shell=/bin/bash + sshkey --username=osbuild "{pubkey}" + rootpw {rootpw} + {rhsm} + eula --agree + # better debug for the sshd failure + bootloader --append="console=ttyS0 systemd.journald.forward_to_console=1" + %post + # workaround for centos-10 as it fails to start here and that causes issue + # with out "check-host-config.sh" that expects a non-degraded boot + systemctl mask mcelog.service || true + {rhsm_unregister} + %end + """)) + new_installer_iso_path = pathlib.Path(tmpdir) / os.path.basename(installer_iso_path) + subprocess.check_call( + ["sudo", "mkksiso", + # Note that we could add: + # systemd.journald.forward_to_console=1 + # here as well but it produces extrem amounts of logs + # that exceeds the gitlab limit + "-c", "console=ttyS0", + "--ks", os.fspath(unattended_ks), + os.fspath(installer_iso_path), new_installer_iso_path]) + return _boot_qemu_iso(arch, new_installer_iso_path, config_file, privkey_path) + + +def boot_qemu_iso(arch, installer_iso_path, config_file): + # We can only test the unattended-iso as the other configs require + # interactive setup of the installer which we do not support in this + # test-runner. + with contextlib.ExitStack() as cm: + # The "unattended-iso" has the CI ssh key, so if we are running + # in CI we can actually do a real image test. Sadly not locally + # because we have no way to log into the installed disk in this + # case, the CI ssh key is secret. + privkey_path = None + if os.environ.get("CI_PRIV_SSH_KEY"): + (privkey_path, _) = cm.enter_context(create_ssh_key()) + return _boot_qemu_iso(arch, installer_iso_path, config_file, privkey_path) + + +def _boot_qemu_iso(arch, installer_iso_path, config_file, privkey_path): + cmd = [BASE_TEST_EXEC+arch, config_file] + make_check_host_config(arch) + # we should pass console=ttyS0 to instaler os that we see install + # progress on the serial console, in the meantime for interactive use + # one cans set the OSBUILD_TEST_QEMU_GUI=1 environment + with contextlib.ExitStack() as cm: + # we need /var/tmp here as /tmp might be on a (small) tmpfs + tmpdir = cm.enter_context(TemporaryDirectory(dir="/var/tmp")) + # create an (empty) target disk, truncate ensures its sparse + # so it will not take up real disk space until things are + # written to it + test_disk_path = pathlib.Path(tmpdir) / "disk.img" + with open(test_disk_path, "w", encoding="utf8") as fp: + fp.truncate(20_000_000_000) + # boot from installer to install to test disk, anaconda will + # reboot automatically for the unattended-iso config. + with QEMU(test_disk_path, cdrom=installer_iso_path) as vm: + vm.start(wait_event="qmp:RESET", snapshot=False, use_ovmf=True, timeout_sec=ISO_BOOT_TIMEOUT) + vm.force_stop() + # now boot test disk and wait for ssh to come up as a minimal boot test + with QEMU(test_disk_path, arch=arch) as vm: + vm.start(use_ovmf=True) + vm.wait_ssh_ready() + if privkey_path: + qemu_cmd_scp_and_run(vm, cmd, privkey_path) + + +def boot_qemu_pxe(arch, pxe_tar_path): + with contextlib.ExitStack() as cm: + # unpack the tar and create a combined image + tmpdir = cm.enter_context(TemporaryDirectory(dir="/var/tmp")) + subprocess.check_call( + ["tar", "-C", tmpdir, "-x", "-f", pxe_tar_path]) + subprocess.check_call( + "echo rootfs.img | cpio -H newc --quiet -L -o > rootfs.cpio", shell=True, cwd=tmpdir) + subprocess.check_call( + "cat initrd.img rootfs.cpio > combined.img", shell=True, cwd=tmpdir) + + # Start an HTTP server to serve the rootfs.img and terminate it after the test. + # Explicitly terminate the HTTP server to avoid blocking on wait(), this cannot + # be done with a context manager for subprocesses. + http_port = get_free_port() + http_server = subprocess.Popen( # pylint: disable=consider-using-with + ["python3", "-m", "http.server", f"{http_port}"], + cwd=tmpdir, + # prevent blocking output + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL + ) + try: + # test disk is unused for live OS + test_disk_path = pathlib.Path(tmpdir) / "disk.img" + with open(test_disk_path, "w", encoding="utf-8") as fp: + fp.truncate(0) + + # test both the combined and HTTP rootfs variants + for use_ovmf in [False, True]: + for root_arg, initrd_file in [ + ("live:/rootfs.img", "combined.img"), + (f"live:http://10.0.2.2:{http_port}/rootfs.img", "initrd.img") + ]: + append_arg = ( + f"rd.live.image root={root_arg} console=ttyS0 " + f"systemd.debug-shell=ttyS0 " + f"systemd.mask=serial-getty@ttyS0.service " + f"systemd.unit=reboot.target" + ) + extra_args = [ + "-kernel", str(pathlib.Path(tmpdir) / "vmlinuz"), + "-initrd", str(pathlib.Path(tmpdir) / initrd_file), + "-append", append_arg + ] + + with QEMU(test_disk_path, memory="3000", arch=arch, extra_args=extra_args) as vm: + # Wait for QMP RESET event instead of SSH since PXE images don't have SSH. + # The systemd.unit=reboot.target will cause a reboot, triggering the RESET event. + vm.start(wait_event="qmp:RESET", snapshot=False, use_ovmf=use_ovmf) + # There is really very little in the rootfs.img (i.e. no ssh, cloud-init + # or other things that open ports or dnf) and we can only control it via the + # kernel commandline. So via the "systemd.unit=reboot.target" kernel commandline + # above we boot and then force a reboot right away as our test. This is not great + # but the best we can do right now. Other options: + # 1. have a blueprint with sshd-server so that we can check for ssh port + # 2. modify vm.py to be able to talk directly to the serial console + # and then run commands directly via that + vm.force_stop() + finally: + http_server.terminate() + http_server.wait() + + +# pylint: disable=too-many-arguments,too-many-positional-arguments +def cmd_boot_aws(arch, image_name, privkey, pubkey, image_path, script_cmd): + make_check_host_config(arch) + aws_config = get_aws_config() + cmd = ["go", "run", "./cmd/boot-aws", "run", + "--access-key-id", aws_config["key_id"], + "--secret-access-key", aws_config["secret_key"], + "--region", aws_config["region"], + "--bucket", aws_config["bucket"], + "--arch", arch, + "--ami-name", image_name, + "--s3-key", f"images/boot/{image_name}", + "--username", "osbuild", + "--ssh-privkey", privkey, + "--ssh-pubkey", pubkey, + image_path, *script_cmd] + runcmd_nc(cmd) + + +def boot_ami(distro, arch, image_type, image_path, config): + cmd = [BASE_TEST_EXEC+arch, config] + make_check_host_config(arch) + with ensure_uncompressed(image_path) as raw_image_path: + with create_ssh_key() as (privkey, pubkey): + image_name = f"image-boot-test-{distro}-{arch}-{image_type}-" + str(uuid.uuid4()) + cmd_boot_aws(arch, image_name, privkey, pubkey, raw_image_path, cmd) + + +# pylint: disable=too-many-arguments,too-many-positional-arguments +def boot_container(distro, arch, image_type, image_path, manifest_id, host_config): + """ + Use bootc-image-builder to build an AMI and boot it. + """ + # push container to registry so we can build it with BIB + # remove when BIB can pull from containers-storage: https://github.com/osbuild/bootc-image-builder/pull/120 + container_name = f"iot-bootable-container:{distro}-{arch}-{manifest_id}" + cmd = ["./tools/ci/push-container.sh", image_path, container_name] + runcmd_nc(cmd) + container_ref = f"{REGISTRY}/{container_name}" + + with TemporaryDirectory() as tmpdir: + with create_ssh_key() as (privkey_file, pubkey_file): + with open(pubkey_file, encoding="utf-8") as pubkey_fp: + pubkey = pubkey_fp.read() + + # write a config to create a user + config_file = os.path.join(tmpdir, "config.json") + with open(config_file, "w", encoding="utf-8") as cfg_fp: + config = { + "blueprint": { + "customizations": { + "user": [ + { + "name": "osbuild", + "key": pubkey, + "groups": [ + "wheel" + ] + } + ] + } + } + } + json.dump(config, cfg_fp) + + # build an AMI + cmd = ["sudo", "podman", "run", + "--rm", "-it", + "--privileged", + "--pull=newer", + "--security-opt", "label=type:unconfined_t", + "-v", f"{tmpdir}:/output", + "-v", f"{config_file}:/config.json", + get_bib_ref(), + "--type=ami", + "--config=/config.json", + container_ref] + runcmd_nc(cmd) + + # boot it + image_name = f"image-boot-test-{distro}-{arch}-{image_type}-" + str(uuid.uuid4()) + + # Build artifacts are owned by root. Make them world accessible. + runcmd(["sudo", "chmod", "a+rwX", "-R", tmpdir]) + raw_image_path = f"{tmpdir}/image/disk.raw" + cmd_boot_aws(arch, image_name, privkey_file, pubkey_file, raw_image_path, + [BASE_TEST_EXEC+arch, host_config]) + + +def boot_vhd(distro, arch, image_path, config): + cmd = [BASE_TEST_EXEC+arch, config] + make_check_host_config(arch) + with ensure_uncompressed(image_path) as raw_image_path: + with create_ssh_key(key_type="rsa") as (privkey, pubkey): + # a lot of resources have <=64 character naming constraint + name = f"{distro}-" + str(uuid.uuid4()) + + az_config = get_azure_config() + cmd = ["go", "run", "./cmd/boot-azure", "run", + "--subscription", az_config["subscription"], + "--tenant", az_config["tenant"], + "--client-id", az_config["client_id"], + "--client-secret", az_config["client_secret"], + "--resource-group", az_config["resource_group"], + "--username", "osbuild", + "--ssh-privkey", privkey, + "--ssh-pubkey", pubkey, + "--vm-name", name, + "--arch", arch, + "--image-name", name, + raw_image_path, *cmd] + runcmd_nc(cmd) + + +def boot_wsl(distro, arch, image_path, config): + with ensure_uncompressed(image_path) as raw_image_path: + cmd = [WSL_TEST_SCRIPT, raw_image_path, BASE_TEST_EXEC+arch, config] + make_check_host_config(arch) + az_config = get_azure_config() + with create_ssh_key(privkey_file=az_config["windows_ssh_privkey"]) as (privkey, pubkey): + # a lot of resources have <=64 character naming constraint + name = f"{distro}-" + str(uuid.uuid4()) + + cmd = ["go", "run", "./cmd/boot-azure", "run", + "--subscription", az_config["subscription"], + "--tenant", az_config["tenant"], + "--client-id", az_config["client_id"], + "--client-secret", az_config["client_secret"], + "--resource-group", az_config["resource_group"], + "--snapshot", az_config["windows_snapshot"], + "--username", "azureuser", + "--ssh-privkey", privkey, + "--ssh-pubkey", pubkey, + "--vm-name", name, + "--arch", arch, + "--size", "Standard_D2as_v5", + raw_image_path, *cmd] + runcmd_nc(cmd) + + +# pylint: disable=too-many-return-statements,too-many-arguments,too-many-positional-arguments +def can_boot_test(manifest_fname, manifest_data, image_type, arch, distro, blueprint): + if image_type not in CAN_BOOT_TEST.get("*", []) + CAN_BOOT_TEST.get(arch, []): + return False + + if image_type in ["image-installer", "minimal-installer"]: + if not blueprint.get("customizations", {}).get("installer", {}).get("unattended"): + print(" not bootable: only unattended installers are supported") + return False + + if image_type in ["network-installer", "everything-network-installer", "server-network-installer"]: + if distro in ["rhel-10.1", "rhel-10.2"]: + print(" not bootable: rhel network-installer tests have incomplete repos in nightly snapshot" + "and won't install") + return False + if distro.startswith("fedora"): + print(" not bootable: fedora network-installer crashes in sshd," + "see https://bugzilla.redhat.com/show_bug.cgi?id=2415883") + return False + if distro == "centos-9": + print(" not bootable: centos-9 will not start an install and waits on source selection") + return False + if distro.startswith("rhel-9"): + print(" not bootable: rhel-9 will not start an install and waits on source selection") + return False + + if image_type in ["qcow2", "generic-qcow2", "cloud-qcow2", "image-installer", "minimal-installer", + "network-installer", "everything-network-installer"]: + if blueprint.get("customizations", {}).get("fips") and distro.startswith("fedora"): + print(" not bootable: fips on fedora is unstable, fails with e.g. dracut:" + "FATAL: FIPS integrity test failed") + return False + # Note that this needs adjustment when we switch to librepo + urls = [src["url"] for src in manifest_data["sources"]["org.osbuild.curl"]["items"].values()] + if not any("ssh-server" in url for url in urls): + # This can happen e.g. when an image is build with the "minimal: true" customization. + # We could use guestfs to inject keys, see PR#1995 + print(f" not bootable: ssh-server not found in manifest {manifest_fname} ({arch} {image_type})") + return False + # We need jq in the image many images do not have it + # (e.g. centos-9/rhel-9 with releasever config) so skip those too + if not any("jq" in url for url in urls): + print(f" not bootable: jq not found in {manifest_fname} ({arch} {image_type})") + return False + + return True diff --git a/test/scripts/imgtestlib/build.py b/test/scripts/imgtestlib/build.py new file mode 100644 index 0000000000..4ab143a636 --- /dev/null +++ b/test/scripts/imgtestlib/build.py @@ -0,0 +1,92 @@ +import json +import os +from typing import Dict + +from .gitlab import print_section_end, print_section_start +from .run import runcmd, runcmd_nc +from .testenv import get_host_distro, get_osbuild_commit, rng_seed_env + + +def build_image(distro, arch, image_type, config_path): + with open(config_path, "r", encoding="utf-8") as config_file: + config = json.load(config_file) + + config_name = config["name"] + + log_section_name = f"build_{distro}_{image_type}_{config_name}" + print_section_start(log_section_name, f"👷 Building image {distro}/{image_type} using config {config_path}") + + # print the config for logging + print(json.dumps(config, indent=2)) + + runcmd(["go", "build", "-o", "./bin/build", "./cmd/build"]) + + cmd = ["sudo", "-E", "./bin/build", "--output", "./build", "--checkpoints", "build", + "--distro", distro, "--arch", arch, "--type", image_type, "--config", config_path] + runcmd_nc(cmd, extra_env=rng_seed_env()) + + print("✅ Build finished!!") + + # Build artifacts are owned by root. Make them world accessible. + runcmd(["sudo", "chmod", "a+rwX", "-R", "./build"]) + + build_dir = os.path.join("build", gen_build_name(distro, arch, image_type, config_name)) + manifest_path = os.path.join(build_dir, "manifest.json") + with open(manifest_path, "r", encoding="utf-8") as manifest_fp: + manifest_data = json.load(manifest_fp) + manifest_id = get_manifest_id(manifest_data) + + osbuild_ver, _ = runcmd(["osbuild", "--version"]) + + distro_version = get_host_distro() + osbuild_commit = get_osbuild_commit(distro_version) + if osbuild_commit is None: + osbuild_commit = "RELEASE" + + build_info = { + "distro": distro, + "arch": arch, + "image-type": image_type, + "config": config_name, + "manifest-checksum": manifest_id, + "osbuild-version": osbuild_ver.decode().strip(), + "osbuild-commit": osbuild_commit, + "commit": os.environ.get("CI_COMMIT_SHA", "N/A"), + "runner-distro": distro_version, + } + write_build_info(build_dir, build_info) + print_section_end(log_section_name) + + +def read_build_info(build_path: str) -> Dict: + """ + Read the info.json file from the build directory and return the data as a dictionary. + """ + info_file_path = os.path.join(build_path, "info.json") + with open(info_file_path, encoding="utf-8") as info_fp: + return json.load(info_fp) + + +def write_build_info(build_path: str, data: Dict): + """ + Write the data to the info.json file in the build directory. + """ + info_file_path = os.path.join(build_path, "info.json") + with open(info_file_path, "w", encoding="utf-8") as info_fp: + json.dump(data, info_fp, indent=2) + + +def get_manifest_id(manifest_data): + md = json.dumps(manifest_data).encode() + out, _ = runcmd(["osbuild", "--inspect", "-"], stdin=md) + data = json.loads(out) + # last stage ID depends on all previous stage IDs, so we can use it as a manifest ID + return data["pipelines"][-1]["stages"][-1]["id"] + + +def gen_build_name(distro, arch, image_type, config_name): + return f"{_u(distro)}-{_u(arch)}-{_u(image_type)}-{_u(config_name)}" + + +def _u(s): + return s.replace("-", "_") diff --git a/test/scripts/imgtestlib/cache.py b/test/scripts/imgtestlib/cache.py new file mode 100644 index 0000000000..8093770bb9 --- /dev/null +++ b/test/scripts/imgtestlib/cache.py @@ -0,0 +1,112 @@ +import os +import subprocess as sp +from datetime import datetime +from typing import List, Optional + +from .run import runcmd_nc +from .testenv import get_host_distro, get_osbuild_commit + +S3_BUCKET = "s3://" + os.environ.get("AWS_BUCKET", "images-ci-cache") +S3_PREFIX = "images/builds" + + +# pylint: disable=too-many-arguments,too-many-positional-arguments +def dl_build_cache( + destination, distro: Optional[str] = None, arch: Optional[str] = None, osbuild_ref: Optional[str] = None, + runner_distro: Optional[str] = None, manifest_id: Optional[str] = None, + include_only: Optional[List[str]] = None): + """ + Downloads image build cache files from the s3 bucket. + + If 'include' is not specified, all files are downloaded. Otherwise, all files will be excluded and the items + in the 'include' list will be passed as '--include' arguments to the 'aws s3 sync' command. + """ + s3url = gen_build_info_s3_dir_path(distro, arch, manifest_id, osbuild_ref, runner_distro) + dl_what = "all files" if include_only is None else "only " + ', '.join(f"'{i}'" for i in include_only) + print(f"⬇️ Downloading {dl_what} from {s3url}") + + # --no-progress wont show progress but will print file list + cmd = ["aws", "s3", "sync", "--no-progress"] + if include_only: + cmd.extend(["--exclude=*"]) + for i in include_only: + cmd.extend([f"--include={i}"]) + cmd.extend([s3url, destination]) + + job = sp.run(cmd, capture_output=True, check=False) + ok = job.returncode == 0 + if not ok: + print(f"⚠️ Failed to sync contents of {s3url}:") + print(job.stdout.decode()) + print(job.stderr.decode()) + return job.stdout.decode(), ok + + +def dl_build_info(destination, distro=None, arch=None, osbuild_ref=None, runner_distro=None): + """ + Downloads all the configs from the s3 bucket. + """ + # only download info.json and bib-* files, otherwise we get manifests and whole images + include = ["*/info.json", "*/bib-*"] + return dl_build_cache(destination, distro, arch, osbuild_ref, runner_distro, include_only=include) + + +def gen_build_info_dir_path_prefix(distro=None, arch=None, manifest_id=None, osbuild_ref=None, runner_distro=None): + """ + Generates the relative path prefix for the location where build info and artifacts will be stored for a specific + build. This is a simple concatenation of the components, but ensures that paths are consistent. The caller is + responsible for prepending the location root to the generated path. + + If no 'osbuild_ref' is specified, the value returned by get_osbuild_commit() for the 'runner_distro' will be used. + if no 'runner_distro' is specified, the value returned by get_host_distro() will be used. + + A fully specified path is returned if all of the 'distro', 'arch' and 'manifest_id' parameters are specified, + otherwise a partial path is returned. Partial path may be useful for working with a superset of build infos. + For a more specific path to be generated when specifying any of the optional parameters, the caller must specify + all of the previous parameters. For example, if 'arch' is specified, 'distro' must also be specified for 'arch' to + be included in the path. + + The returned path always has a trailing separator at the end to signal that it is a directory. + """ + if runner_distro is None: + runner_distro = get_host_distro() + if osbuild_ref is None: + osbuild_ref = get_osbuild_commit(runner_distro) + + path = os.path.join(f"osbuild-ref-{osbuild_ref}", f"runner-{runner_distro}") + for p in (distro, arch, f"manifest-id-{manifest_id}" if manifest_id else None): + if p is None: + return path + "/" + path = os.path.join(path, p) + return path + "/" + + +def gen_build_info_s3_dir_path(distro=None, arch=None, manifest_id=None, osbuild_ref=None, runner_distro=None): + """ + Generates the s3 URL for the location where build info and artifacts will be stored for a specific + one or more builds, depending on the parameters specified. + + A fully specified path is returned if all parameters are specified, otherwise a partial path is returned. + This function basically just prepends the S3_BUCKET and S3_PREFIX to the path generated by + gen_build_info_dir_path_prefix(). + """ + return os.path.join( + S3_BUCKET, + S3_PREFIX, + gen_build_info_dir_path_prefix(distro, arch, manifest_id, osbuild_ref, runner_distro), + ) + + +def touch_s3(distro, arch, manifest_id, osbuild_ref=None, runner_distro=None): + """ + Update the timestamps of a path in S3 by adding a metadata field to each file recursively. This can be used to + "freshen up" relevant files in the build cache so that images that are still current but haven't been updated in a + while don't get garbage collected. + """ + # NOTE: we can make this async - updating the objects can take a few seconds and can be done in parallel + s3url = gen_build_info_s3_dir_path(distro, arch, manifest_id, osbuild_ref, runner_distro) + # the exact key and value don't matter, but let's add the current datetime to make it a bit more meaningful + now = str(datetime.now()) + print(f"⌚ Updating timestamps for {s3url} ({now})") + cmd = ["aws", "s3", "cp", "--recursive", "--metadata", f"touched={now}", s3url, s3url] + runcmd_nc(cmd) diff --git a/test/scripts/imgtestlib/core.py b/test/scripts/imgtestlib/core.py new file mode 100644 index 0000000000..4db9c5e5ad --- /dev/null +++ b/test/scripts/imgtestlib/core.py @@ -0,0 +1,371 @@ +import argparse +import json +import os +import pathlib +import sys +from glob import glob +from typing import Dict + +from .boot import can_boot_test +from .build import get_manifest_id +from .cache import dl_build_info, gen_build_info_dir_path_prefix, touch_s3 +from .gitlab import print_section_end, print_section_start +from .run import runcmd +from .testenv import get_bib_ref, host_container_arch, rng_seed_env + +TEST_CACHE_ROOT = ".cache/osbuild-images" +CONFIGS_PATH = "./test/configs" +CONFIG_LIST = "./test/config-list.json" + +BIB_TYPES = [ + "iot-bootable-container" +] + + +# base and terraform bits copied from main .gitlab-ci.yml +# needed for status reporting and defining the runners +BASE_CONFIG = """ +.base: + before_script: + - cat schutzbot/team_ssh_keys.txt | + tee -a ~/.ssh/authorized_keys > /dev/null + interruptible: true + retry: 1 + tags: + - terraform + variables: + PYTHONUNBUFFERED: 1 + +.terraform: + extends: .base + tags: + - terraform +""" + +NULL_CONFIG = """ +NullBuild: + stage: test + script: echo "No manifest changes detected. Skipping build." + tags: + - "shell" +""" + + +def list_images(distros=None, arches=None, images=None): + distros_arg = "*" + if distros: + distros_arg = ",".join(distros) + arches_arg = "*" + if arches: + arches_arg = ",".join(arches) + images_arg = "*" + if images: + images_arg = ",".join(images) + env = {"GOPROXY": "https://proxy.golang.org,direct"} + out, _ = runcmd(["go", "run", "./cmd/list-images", "--json", + "--distros", distros_arg, "--arches", arches_arg, "--types", images_arg], extra_env=env) + return json.loads(out) + + +def check_config_names(): + """ + Check that all the configs we rely on have names that match the file name, otherwise the test skipping and pipeline + generation will be incorrect. + """ + bad_configs = [] + for file in pathlib.Path(CONFIGS_PATH).glob("*.json"): + config = json.loads(file.read_text()) + if file.stem != config["name"]: + bad_configs.append(str(file)) + + if bad_configs: + print("☠️ ERROR: The following test configs have names that don't match their filenames.") + print("\n".join(bad_configs)) + print("This will produce incorrect test generation and results.") + print("Aborting.") + sys.exit(1) + + +def gen_manifests(outputdir, config_list=None, distros=None, arches=None, images=None, + commits=False, flatpaks=False, skip_no_config=False): + # pylint: disable=too-many-arguments,too-many-positional-arguments + cmd = ["go", "run", "./cmd/gen-manifests", + "--cache", os.path.join(TEST_CACHE_ROOT, "rpmmd"), + "--output", outputdir, + "--workers", "100"] + if config_list: + cmd.extend(["--config-list", config_list]) + if distros: + cmd.extend(["--distros", ",".join(distros)]) + if arches: + cmd.extend(["--arches", ",".join(arches)]) + if images: + cmd.extend(["--types", ",".join(images)]) + if commits: + cmd.append("--commits") + if flatpaks: + cmd.append("--flatpaks") + if skip_no_config: + cmd.append("--skip-noconfig") + env = rng_seed_env() + env["GOPROXY"] = "https://proxy.golang.org,direct" + print("⌨️" + " ".join(cmd) + " ENV: " + str(env)) + _, stderr = runcmd(cmd, extra_env=env) + return stderr + + +def read_manifests(path): + """ + Read all manifests in the given path, calculate their IDs, and return a dictionary mapping each filename to the data + and its ID. + """ + print(f"📖 Reading manifests in {path}") + manifests = {} + for manifest_fname in os.listdir(path): + manifest_path = os.path.join(path, manifest_fname) + with open(manifest_path, encoding="utf-8") as manifest_file: + manifest_data = json.load(manifest_file) + manifests[manifest_fname] = { + "data": manifest_data, + "id": get_manifest_id(manifest_data["manifest"]), + } + print("✅ Done") + return manifests + + +# pylint: disable=too-many-branches +def check_for_build(manifest_fname, build_request, manifest_data, build_info_dir, errors): + """ + Checks if a manifest was built (and optionally booted) successfully. + + This function returns True if the image needs to be built. + """ + build_info_path = os.path.join(build_info_dir, "info.json") + # rebuild if matching build info is not found + if not os.path.exists(build_info_path): + print(f"Build info not found: {build_info_path}") + print(" Adding config to build pipeline.") + return True + + try: + with open(build_info_path, encoding="utf-8") as build_info_fp: + dl_config = json.load(build_info_fp) + except json.JSONDecodeError as jd: + errors.append(( + f"failed to parse {build_info_path}\n" + f"{jd.msg}\n" + )) + print(" Adding config to build pipeline.") + return True + + commit = dl_config["commit"] + pr = dl_config.get("pr") + url = f"https://github.com/osbuild/images/commit/{commit}" + print(f"🖼️ Manifest {manifest_fname} was successfully built in commit {commit}\n {url}") + if "gh-readonly-queue" in pr: + print(f" This commit was on a merge queue: {pr}") + elif pr: + print(f" PR-{pr}: https://github.com/osbuild/images/pull/{pr}") + else: + print(" No PR/branch info available") + + image_type = dl_config["image-type"] + if not can_boot_test(manifest_fname, manifest_data, build_request["image-type"], build_request["arch"], + build_request["distro"], build_request["config"].get("blueprint", {})): + print(f" Boot testing for {image_type} is not yet supported") + return False + + # boot testing supported: check if it's been tested, otherwise queue it for rebuild and boot + if dl_config.get("boot-success", False): + print(" This image was successfully boot tested") + + # check if it's a BIB type and compare image IDs + if image_type in BIB_TYPES: + # Successful boot tests with BIB add a file to the directory as bib-. Collect them and compare. + bib_ids = glob("bib-*", root_dir=build_info_dir) + # add the _old_ bib ID that we used to keep in the info.json + config_bib_id = dl_config.get("bib-id") + if config_bib_id: + bib_ids.append(f"bib-{config_bib_id}") + bib_ref = get_bib_ref() + current_id = skopeo_inspect_id(f"docker://{bib_ref}", host_container_arch()) + if f"bib-{current_id}" not in bib_ids: + if bib_ids: + print(" Container disk image was built with the following bootc-image-builder images:") + print(" - " + "\n -".join(bib_ids)) + else: + print(" No bib IDs found.") + print(f" Testing {current_id}") + print(" Adding config to build pipeline.") + return True + + return False + print(" Boot test success not found.") + + # default to build + print(" Adding config to build pipeline.") + return True + + +def filter_builds(manifests, distro=None, arch=None, skip_ostree_pull=True): + """ + Returns a list of build requests for the manifests that have no matching config in the test build cache. + """ + print_section_start("filter-build-configs", f"⚙️ Filtering {len(manifests)} build configurations") + dl_root_path = os.path.join(TEST_CACHE_ROOT, "s3configs", "builds") + dl_path = os.path.join(dl_root_path, gen_build_info_dir_path_prefix(distro, arch)) + os.makedirs(dl_path, exist_ok=True) + build_requests = [] + + out, dl_ok = dl_build_info(dl_path, distro, arch) + # continue even if the dl failed; will build all configs + if dl_ok: + # print output which includes list of downloaded files for CI job log + print(out) + + errors: list[str] = [] + for manifest_fname, data in manifests.items(): + manifest_id = data["id"] + data = data.get("data") + build_request = data["build-request"] + distro = build_request["distro"] + arch = build_request["arch"] + image_type = build_request["image-type"] + config = build_request["config"] + config_name = config["name"] + options = config.get("options", {}) + + # check if the config specifies an ostree URL and skip it if requested + if skip_ostree_pull and options.get("ostree", {}).get("url"): + print(f"🦘 Skipping {distro}/{arch}/{image_type}/{config_name} (ostree dependency)") + continue + + # add manifest id to build request + build_request["manifest-checksum"] = manifest_id + + # check if the hash_fname exists in the synced directory + build_info_dir = os.path.join( + dl_root_path, + gen_build_info_dir_path_prefix(distro, arch, manifest_id) + ) + + if check_for_build(manifest_fname, build_request, data["manifest"], build_info_dir, errors): + build_requests.append(build_request) + else: + # The specific build configuration exists in the cache and wont be rebuilt. Update the file timestamps to + # keep them fresh in the cache. + touch_s3(distro, arch, manifest_id) + + print("✅ Config filtering done!\n") + if errors: + # print errors at the end so they're visible + print("⚠️ Errors:") + print("\n".join(errors)) + + print_section_end("filter-build-configs") + return build_requests + + +def clargs(): + default_arch = os.uname().machine + parser = argparse.ArgumentParser() + parser.add_argument("config", type=str, help="path to write config") + parser.add_argument("--distro", type=str, required=True, + help="distro to generate configs for") + parser.add_argument("--arch", type=str, default=default_arch, + help="architecture to generate configs for (defaults to host architecture)") + + return parser + + +def is_manifest_list(data): + """Inspect a manifest determine if it's a multi-image manifest-list.""" + media_type = data.get("mediaType") + # Check if mediaType is set according to docker or oci specifications + if media_type in ("application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.index.v1+json"): + return True + + # According to the OCI spec, setting mediaType is not mandatory. So, if it is not set at all, check for the + # existence of manifests + if media_type is None and data.get("manifests") is not None: + return True + + return False + + +def skopeo_inspect_id(image_name: str, arch: str) -> str: + """ + Returns the image ID (config digest) of the container image. If the image resolves to a manifest list, the config + digest of the given architecture is resolved. + + Runs with 'sudo' when inspecting a local container because in our tests we need to read the root container storage. + """ + cmd = ["skopeo", "inspect", "--raw", image_name] + if image_name.startswith("containers-storage"): + cmd = ["sudo"] + cmd + out, _ = runcmd(cmd) + data = json.loads(out) + if not is_manifest_list(data): + return data["config"]["digest"] + + for manifest in data.get("manifests", []): + platform = manifest.get("platform", {}) + img_arch = platform.get("architecture", "") + img_ostype = platform.get("os", "") + + if arch != img_arch or img_ostype != "linux": + continue + + if "@" in image_name: + image_no_tag = image_name.split("@")[0] + else: + image_no_tag = ":".join(image_name.split(":")[:-1]) + manifest_digest = manifest["digest"] + arch_image_name = f"{image_no_tag}@{manifest_digest}" + # inspect the arch-specific manifest to get the image ID (config digest) + return skopeo_inspect_id(arch_image_name, arch) + + # don't error out, just return an empty string and let the caller handle it + return "" + + +def get_tag_for(runner): + if runner.startswith("aws/"): + return "terraform" + if runner.startswith("rhos-01/"): + return "terraform/openstack" + + raise ValueError(f"Unknown runner: {runner}") + + +def find_image_file(build_path: str) -> str: + """ + Find the path to the image by reading the manifest to get the name of the last pipeline and searching for the file + under the directory named after the pipeline. Raises RuntimeError if no or multiple files are found in the expected + path. + """ + manifest_file = os.path.join(build_path, "manifest.json") + with open(manifest_file, encoding="utf-8") as manifest: + data = json.load(manifest) + + last_pipeline = data["pipelines"][-1]["name"] + files = os.listdir(os.path.join(build_path, last_pipeline)) + if len(files) > 1: + error = "Multiple files found in build path while searching for image file" + error += "\n".join(files) + raise RuntimeError(error) + + if len(files) == 0: + raise RuntimeError("No found in build path while searching for image file") + + return os.path.join(build_path, last_pipeline, files[0]) + + +def read_manifest(build_path: str) -> Dict: + """ + Read the manifest.json file from the build directory and return the data as a dictionary. + """ + info_file_path = os.path.join(build_path, "manifest.json") + with open(info_file_path, encoding="utf-8") as info_fp: + return json.load(info_fp) diff --git a/test/scripts/imgtestlib/gitlab.py b/test/scripts/imgtestlib/gitlab.py new file mode 100644 index 0000000000..b911123b23 --- /dev/null +++ b/test/scripts/imgtestlib/gitlab.py @@ -0,0 +1,37 @@ +import os +from datetime import datetime + + +def running_in_gitlab(): + """ + Returns true if running in GitLab CI. + """ + return os.environ.get("GITLAB_CI") + + +def print_section_start(name: str, msg: str = ""): + """ + Prints a section header with a timestamp for logging output during tests. + If running in GitLab CI, it also creates a collapsible section. + + https://docs.gitlab.com/ci/jobs/job_logs/#custom-collapsible-sections + """ + now = datetime.now() + if running_in_gitlab(): + print(f"\033[0Ksection_start:{int(now.timestamp())}:{name}[collapsed=true]\r\033[0K{msg}") + return + + # custom line for non CI runs + isonow = now.isoformat() + print(f":: [{isonow}] {msg} ({name})") + + +def print_section_end(name: str): + now = datetime.now() + if running_in_gitlab(): + print(f"\033[0Ksection_end:{int(now.timestamp())}:{name}\r\033[0K") + return + + # custom line for non CI runs + isonow = now.isoformat() + print(f":: [{isonow}] Done ({name})") diff --git a/test/scripts/imgtestlib/imgtestlib.py b/test/scripts/imgtestlib/imgtestlib.py deleted file mode 100644 index 436c305c61..0000000000 --- a/test/scripts/imgtestlib/imgtestlib.py +++ /dev/null @@ -1,839 +0,0 @@ -import argparse -import json -import os -import pathlib -import subprocess as sp -import sys -from datetime import datetime -from glob import glob -from typing import Dict, List, Optional - -TEST_CACHE_ROOT = ".cache/osbuild-images" -CONFIGS_PATH = "./test/configs" -CONFIG_LIST = "./test/config-list.json" - -S3_BUCKET = "s3://" + os.environ.get("AWS_BUCKET", "images-ci-cache") -S3_PREFIX = "images/builds" - -REGISTRY = "registry.gitlab.com/redhat/services/products/image-builder/ci/images" - -# Path to the Schutzfile relative to the root of the repository -SCHUTZFILE = str(pathlib.Path(__file__).resolve().parents[3] / "Schutzfile") -OS_RELEASE_FILE = "/etc/os-release" - -# image types that can be boot tested -# Keep in sync with test/scripts/boot-image which has the same checks again -CAN_BOOT_TEST = { - "*": [ - "ami", - "ec2", - "ec2-ha", - "ec2-sap", - "edge-ami", - "iot-bootable-container", - "vhd", - "cloud-ec2", - ], - "x86_64": [ - "image-installer", "minimal-installer", "network-installer", - "qcow2", "generic-qcow2", "cloud-qcow2", - "wsl", "generic-wsl", - "bootc-generic-iso", - ] -} - -BIB_TYPES = [ - "iot-bootable-container" -] - - -# base and terraform bits copied from main .gitlab-ci.yml -# needed for status reporting and defining the runners -BASE_CONFIG = """ -.base: - before_script: - - cat schutzbot/team_ssh_keys.txt | - tee -a ~/.ssh/authorized_keys > /dev/null - interruptible: true - retry: 1 - tags: - - terraform - variables: - PYTHONUNBUFFERED: 1 - -.terraform: - extends: .base - tags: - - terraform -""" - -NULL_CONFIG = """ -NullBuild: - stage: test - script: echo "No manifest changes detected. Skipping build." - tags: - - "shell" -""" - - -def runcmd(cmd, stdin=None, extra_env=None, capture_output=True): - """ - Run the cmd using sp.run() and exit with the returncode if it is non-zero. - Output is captured and both stdout and stderr are returned if the run succeeds. - If it fails, the output is printed before exiting. - """ - env = None - if extra_env: - env = os.environ - env.update(extra_env) - job = sp.run(cmd, input=stdin, capture_output=capture_output, env=env, check=False) - if job.returncode > 0: - print(f"❌ Command failed: {cmd}") - if job.stdout: - print(job.stdout.decode()) - if job.stderr: - print(job.stderr.decode()) - sys.exit(job.returncode) - - return job.stdout, job.stderr - - -def runcmd_nc(cmd, stdin=None, extra_env=None): - """ - Run the cmd using sp.run() and exit with the returncode if it is non-zero. - Output it not captured. - """ - runcmd(cmd, stdin=stdin, extra_env=extra_env, capture_output=False) - - -def list_images(distros=None, arches=None, images=None): - distros_arg = "*" - if distros: - distros_arg = ",".join(distros) - arches_arg = "*" - if arches: - arches_arg = ",".join(arches) - images_arg = "*" - if images: - images_arg = ",".join(images) - env = {"GOPROXY": "https://proxy.golang.org,direct"} - out, _ = runcmd(["go", "run", "./cmd/list-images", "--json", - "--distros", distros_arg, "--arches", arches_arg, "--types", images_arg], extra_env=env) - return json.loads(out) - - -# pylint: disable=too-many-arguments,too-many-positional-arguments -def dl_build_cache( - destination, distro: Optional[str] = None, arch: Optional[str] = None, osbuild_ref: Optional[str] = None, - runner_distro: Optional[str] = None, manifest_id: Optional[str] = None, - include_only: Optional[List[str]] = None): - """ - Downloads image build cache files from the s3 bucket. - - If 'include' is not specified, all files are downloaded. Otherwise, all files will be excluded and the items - in the 'include' list will be passed as '--include' arguments to the 'aws s3 sync' command. - """ - s3url = gen_build_info_s3_dir_path(distro, arch, manifest_id, osbuild_ref, runner_distro) - dl_what = "all files" if include_only is None else "only " + ', '.join(f"'{i}'" for i in include_only) - print(f"⬇️ Downloading {dl_what} from {s3url}") - - cmd = [ - "aws", "s3", "sync", - "--no-progress", # wont show progress but will print file list - ] - if include_only: - cmd.extend(["--exclude=*"]) - for i in include_only: - cmd.extend([f"--include={i}"]) - cmd.extend([s3url, destination]) - - job = sp.run(cmd, capture_output=True, check=False) - ok = job.returncode == 0 - if not ok: - print(f"⚠️ Failed to sync contents of {s3url}:") - print(job.stdout.decode()) - print(job.stderr.decode()) - return job.stdout.decode(), ok - - -def dl_build_info(destination, distro=None, arch=None, osbuild_ref=None, runner_distro=None): - """ - Downloads all the configs from the s3 bucket. - """ - # only download info.json and bib-* files, otherwise we get manifests and whole images - include = ["*/info.json", "*/bib-*"] - return dl_build_cache(destination, distro, arch, osbuild_ref, runner_distro, include_only=include) - - -def get_manifest_id(manifest_data): - md = json.dumps(manifest_data).encode() - out, _ = runcmd(["osbuild", "--inspect", "-"], stdin=md) - data = json.loads(out) - # last stage ID depends on all previous stage IDs, so we can use it as a manifest ID - return data["pipelines"][-1]["stages"][-1]["id"] - - -def _u(s): - return s.replace("-", "_") - - -def gen_build_name(distro, arch, image_type, config_name): - return f"{_u(distro)}-{_u(arch)}-{_u(image_type)}-{_u(config_name)}" - - -def gen_build_info_dir_path_prefix(distro=None, arch=None, manifest_id=None, osbuild_ref=None, runner_distro=None): - """ - Generates the relative path prefix for the location where build info and artifacts will be stored for a specific - build. This is a simple concatenation of the components, but ensures that paths are consistent. The caller is - responsible for prepending the location root to the generated path. - - If no 'osbuild_ref' is specified, the value returned by get_osbuild_commit() for the 'runner_distro' will be used. - if no 'runner_distro' is specified, the value returned by get_host_distro() will be used. - - A fully specified path is returned if all of the 'distro', 'arch' and 'manifest_id' parameters are specified, - otherwise a partial path is returned. Partial path may be useful for working with a superset of build infos. - For a more specific path to be generated when specifying any of the optional parameters, the caller must specify - all of the previous parameters. For example, if 'arch' is specified, 'distro' must also be specified for 'arch' to - be included in the path. - - The returned path always has a trailing separator at the end to signal that it is a directory. - """ - if runner_distro is None: - runner_distro = get_host_distro() - if osbuild_ref is None: - osbuild_ref = get_osbuild_commit(runner_distro) - - path = os.path.join(f"osbuild-ref-{osbuild_ref}", f"runner-{runner_distro}") - for p in (distro, arch, f"manifest-id-{manifest_id}" if manifest_id else None): - if p is None: - return path + "/" - path = os.path.join(path, p) - return path + "/" - - -def gen_build_info_s3_dir_path(distro=None, arch=None, manifest_id=None, osbuild_ref=None, runner_distro=None): - """ - Generates the s3 URL for the location where build info and artifacts will be stored for a specific - one or more builds, depending on the parameters specified. - - A fully specified path is returned if all parameters are specified, otherwise a partial path is returned. - This function basically just prepends the S3_BUCKET and S3_PREFIX to the path generated by - gen_build_info_dir_path_prefix(). - """ - return os.path.join( - S3_BUCKET, - S3_PREFIX, - gen_build_info_dir_path_prefix(distro, arch, manifest_id, osbuild_ref, runner_distro), - ) - - -def check_config_names(): - """ - Check that all the configs we rely on have names that match the file name, otherwise the test skipping and pipeline - generation will be incorrect. - """ - bad_configs = [] - for file in pathlib.Path(CONFIGS_PATH).glob("*.json"): - config = json.loads(file.read_text()) - if file.stem != config["name"]: - bad_configs.append(str(file)) - - if bad_configs: - print("☠️ ERROR: The following test configs have names that don't match their filenames.") - print("\n".join(bad_configs)) - print("This will produce incorrect test generation and results.") - print("Aborting.") - sys.exit(1) - - -def gen_manifests(outputdir, config_list=None, distros=None, arches=None, images=None, - commits=False, flatpaks=False, skip_no_config=False): - # pylint: disable=too-many-arguments,too-many-positional-arguments - cmd = ["go", "run", "./cmd/gen-manifests", - "--cache", os.path.join(TEST_CACHE_ROOT, "rpmmd"), - "--output", outputdir, - "--workers", "100"] - if config_list: - cmd.extend(["--config-list", config_list]) - if distros: - cmd.extend(["--distros", ",".join(distros)]) - if arches: - cmd.extend(["--arches", ",".join(arches)]) - if images: - cmd.extend(["--types", ",".join(images)]) - if commits: - cmd.append("--commits") - if flatpaks: - cmd.append("--flatpaks") - if skip_no_config: - cmd.append("--skip-noconfig") - env = rng_seed_env() - env["GOPROXY"] = "https://proxy.golang.org,direct" - print("⌨️" + " ".join(cmd) + " ENV: " + str(env)) - _, stderr = runcmd(cmd, extra_env=env) - return stderr - - -def read_manifests(path): - """ - Read all manifests in the given path, calculate their IDs, and return a dictionary mapping each filename to the data - and its ID. - """ - print(f"📖 Reading manifests in {path}") - manifests = {} - for manifest_fname in os.listdir(path): - manifest_path = os.path.join(path, manifest_fname) - with open(manifest_path, encoding="utf-8") as manifest_file: - manifest_data = json.load(manifest_file) - manifests[manifest_fname] = { - "data": manifest_data, - "id": get_manifest_id(manifest_data["manifest"]), - } - print("✅ Done") - return manifests - - -def _is_bootc_manifest(manifest_data): - """ - Check if the manifest is a bootc manifest by looking for the - `org.osbuild.bootc.install-to-filesystem` stage in any of the pipelines - other than the build pipeline. - """ - for pipeline in manifest_data.get("pipelines", []): - if pipeline.get("name") == "build": - continue - for stage in pipeline.get("stages", []): - if stage.get("type") == "org.osbuild.bootc.install-to-filesystem": - return True - return False - - -# pylint: disable=too-many-return-statements,too-many-branches -def build_image(distro, arch, image_type, config_path): - with open(config_path, "r", encoding="utf-8") as config_file: - config = json.load(config_file) - - config_name = config["name"] - - log_section_name = f"build_{distro}_{image_type}_{config_name}" - print_section_start(log_section_name, f"👷 Building image {distro}/{image_type} using config {config_path}") - - # print the config for logging - print(json.dumps(config, indent=2)) - - runcmd(["go", "build", "-o", "./bin/build", "./cmd/build"]) - - cmd = ["sudo", "-E", "./bin/build", "--output", "./build", "--checkpoints", "build", - "--distro", distro, "--arch", arch, "--type", image_type, "--config", config_path] - runcmd_nc(cmd, extra_env=rng_seed_env()) - - print("✅ Build finished!!") - - # Build artifacts are owned by root. Make them world accessible. - runcmd(["sudo", "chmod", "a+rwX", "-R", "./build"]) - - build_dir = os.path.join("build", gen_build_name(distro, arch, image_type, config_name)) - manifest_path = os.path.join(build_dir, "manifest.json") - with open(manifest_path, "r", encoding="utf-8") as manifest_fp: - manifest_data = json.load(manifest_fp) - manifest_id = get_manifest_id(manifest_data) - - osbuild_ver, _ = runcmd(["osbuild", "--version"]) - - distro_version = get_host_distro() - osbuild_commit = get_osbuild_commit(distro_version) - if osbuild_commit is None: - osbuild_commit = "RELEASE" - - build_info = { - "distro": distro, - "arch": arch, - "image-type": image_type, - "config": config_name, - "manifest-checksum": manifest_id, - "osbuild-version": osbuild_ver.decode().strip(), - "osbuild-commit": osbuild_commit, - "commit": os.environ.get("CI_COMMIT_SHA", "N/A"), - "runner-distro": distro_version, - } - write_build_info(build_dir, build_info) - print_section_end(log_section_name) - - -# pylint: disable=too-many-return-statements -def can_boot_test(manifest_fname, manifest_data, image_type, arch, distro, blueprint): - if image_type not in CAN_BOOT_TEST.get("*", []) + CAN_BOOT_TEST.get(arch, []): - return False - - if image_type in ["image-installer", "minimal-installer"]: - if not blueprint.get("customizations", {}).get("installer", {}).get("unattended"): - print(" not bootable: only unattended installers are supported") - return False - - if image_type in ["network-installer", "everything-network-installer", "server-network-installer"]: - if distro in ["rhel-10.1", "rhel-10.2"]: - print(" not bootable: rhel network-installer tests have incomplete repos in nightly snapshot" - "and won't install") - return False - if distro.startswith("fedora"): - print(" not bootable: fedora network-installer crashes in sshd," - "see https://bugzilla.redhat.com/show_bug.cgi?id=2415883") - return False - if distro == "centos-9": - print(" not bootable: centos-9 will not start an install and waits on source selection") - return False - if distro.startswith("rhel-9"): - print(" not bootable: rhel-9 will not start an install and waits on source selection") - return False - - if image_type in ["qcow2", "generic-qcow2", "cloud-qcow2", "image-installer", "minimal-installer", - "network-installer", "everything-network-installer", "bootc-generic-iso"]: - if blueprint.get("customizations", {}).get("fips") and distro.startswith("fedora"): - print(" not bootable: fips on fedora is unstable, fails with e.g. dracut:" - "FATAL: FIPS integrity test failed") - return False - if not image_type.startswith("bootc-") and not _is_bootc_manifest(manifest_data): - # Note that this needs adjustment when we switch to librepo - urls = [src["url"] for src in manifest_data["sources"]["org.osbuild.curl"]["items"].values()] - if not any("ssh-server" in url for url in urls): - # This can happen e.g. when an image is build with the "minimal: true" customization. - # We could use guestfs to inject keys, see PR#1995 - print(f" not bootable: ssh-server not found in manifest {manifest_fname} ({arch} {image_type})") - return False - # We need jq in the image many images do not have it - # (e.g. centos-9/rhel-9 with releasever config) so skip those too - if not any("jq" in url for url in urls): - print(f" not bootable: jq not found in {manifest_fname} ({arch} {image_type})") - return False - - return True - - -# pylint: disable=too-many-branches -def check_for_build(manifest_fname, build_request, manifest_data, build_info_dir, errors): - """ - Checks if a manifest was built (and optionally booted) successfully. - - This function returns True if the image needs to be built. - """ - build_info_path = os.path.join(build_info_dir, "info.json") - # rebuild if matching build info is not found - if not os.path.exists(build_info_path): - print(f"Build info not found: {build_info_path}") - print(" Adding config to build pipeline.") - return True - - try: - with open(build_info_path, encoding="utf-8") as build_info_fp: - dl_config = json.load(build_info_fp) - except json.JSONDecodeError as jd: - errors.append(( - f"failed to parse {build_info_path}\n" - f"{jd.msg}\n" - )) - print(" Adding config to build pipeline.") - return True - - commit = dl_config["commit"] - pr = dl_config.get("pr") - url = f"https://github.com/osbuild/images/commit/{commit}" - print(f"🖼️ Manifest {manifest_fname} was successfully built in commit {commit}\n {url}") - if "gh-readonly-queue" in pr: - print(f" This commit was on a merge queue: {pr}") - elif pr: - print(f" PR-{pr}: https://github.com/osbuild/images/pull/{pr}") - else: - print(" No PR/branch info available") - - image_type = dl_config["image-type"] - if not can_boot_test(manifest_fname, manifest_data, build_request["image-type"], build_request["arch"], - build_request["distro"], build_request["config"].get("blueprint", {})): - print(f" Boot testing for {image_type} is not yet supported") - return False - - # boot testing supported: check if it's been tested, otherwise queue it for rebuild and boot - if dl_config.get("boot-success", False): - print(" This image was successfully boot tested") - - # check if it's a BIB type and compare image IDs - if image_type in BIB_TYPES: - # Successful boot tests with BIB add a file to the directory as bib-. Collect them and compare. - bib_ids = glob("bib-*", root_dir=build_info_dir) - # add the _old_ bib ID that we used to keep in the info.json - config_bib_id = dl_config.get("bib-id") - if config_bib_id: - bib_ids.append(f"bib-{config_bib_id}") - bib_ref = get_bib_ref() - current_id = skopeo_inspect_id(f"docker://{bib_ref}", host_container_arch()) - if f"bib-{current_id}" not in bib_ids: - if bib_ids: - print(" Container disk image was built with the following bootc-image-builder images:") - print(" - " + "\n -".join(bib_ids)) - else: - print(" No bib IDs found.") - print(f" Testing {current_id}") - print(" Adding config to build pipeline.") - return True - - return False - print(" Boot test success not found.") - - # default to build - print(" Adding config to build pipeline.") - return True - - -def filter_builds(manifests, distro=None, arch=None, skip_ostree_pull=True): - """ - Returns a list of build requests for the manifests that have no matching config in the test build cache. - """ - print_section_start("filter-build-configs", f"⚙️ Filtering {len(manifests)} build configurations") - dl_root_path = os.path.join(TEST_CACHE_ROOT, "s3configs", "builds") - dl_path = os.path.join(dl_root_path, gen_build_info_dir_path_prefix(distro, arch)) - os.makedirs(dl_path, exist_ok=True) - build_requests = [] - - out, dl_ok = dl_build_info(dl_path, distro, arch) - # continue even if the dl failed; will build all configs - if dl_ok: - # print output which includes list of downloaded files for CI job log - print(out) - - errors: list[str] = [] - for manifest_fname, data in manifests.items(): - manifest_id = data["id"] - data = data.get("data") - build_request = data["build-request"] - distro = build_request["distro"] - arch = build_request["arch"] - image_type = build_request["image-type"] - config = build_request["config"] - config_name = config["name"] - options = config.get("options", {}) - - # check if the config specifies an ostree URL and skip it if requested - if skip_ostree_pull and options.get("ostree", {}).get("url"): - print(f"🦘 Skipping {distro}/{arch}/{image_type}/{config_name} (ostree dependency)") - continue - - # add manifest id to build request - build_request["manifest-checksum"] = manifest_id - - # check if the hash_fname exists in the synced directory - build_info_dir = os.path.join( - dl_root_path, - gen_build_info_dir_path_prefix(distro, arch, manifest_id) - ) - - if check_for_build(manifest_fname, build_request, data["manifest"], build_info_dir, errors): - build_requests.append(build_request) - else: - # The specific build configuration exists in the cache and wont be rebuilt. Update the file timestamps to - # keep them fresh in the cache. - touch_s3(distro, arch, manifest_id) - - print("✅ Config filtering done!\n") - if errors: - # print errors at the end so they're visible - print("⚠️ Errors:") - print("\n".join(errors)) - - print_section_end("filter-build-configs") - return build_requests - - -def clargs(): - default_arch = os.uname().machine - parser = argparse.ArgumentParser() - parser.add_argument("config", type=str, help="path to write config") - parser.add_argument("--distro", type=str, required=True, - help="distro to generate configs for") - parser.add_argument("--arch", type=str, default=default_arch, - help="architecture to generate configs for (defaults to host architecture)") - - return parser - - -def read_osrelease(): - """Read Operating System Information from `os-release` - - This creates a dictionary with information describing the running operating system. It reads the information from - the path array provided as `paths`. The first available file takes precedence. It must be formatted according to - the rules in `os-release(5)`. - """ - osrelease = {} - - with open(OS_RELEASE_FILE, encoding="utf8") as orf: - for line in orf: - line = line.strip() - if not line: - continue - if line[0] == "#": - continue - key, value = line.split("=", 1) - osrelease[key] = value.strip('"') - - return osrelease - - -def get_host_distro(): - """ - Get the host distro version based on data in the os-release file. - The format is - (e.g. fedora-41). - - Can be overridden by setting the OSBUILD_IMGTESTLIB_HOST_DISTRO env var. - """ - # overriding this is useful for running tests locally on any distro version while still being able to reuse the - # cached images from the CI runners - if distro := os.environ.get("OSBUILD_IMGTESTLIB_HOST_DISTRO"): - return distro - osrelease = read_osrelease() - return f"{osrelease['ID']}-{osrelease['VERSION_ID']}" - - -def get_osbuild_commit(distro_version): - """ - Get the osbuild commit defined in the Schutzfile for the host distro or common. - If not set, returns None. - """ - with open(SCHUTZFILE, encoding="utf-8") as schutzfile: - data = json.load(schutzfile) - - commit = data.get(distro_version, {}).get("dependencies", {}).get("osbuild", {}).get("commit", None) - if commit is None: - commit = data.get("common", {}).get("dependencies", {}).get("osbuild", {}).get("commit", None) - return commit - - -def get_bib_ref(): - """ - Get the bootc-image-builder ref defined in the Schutzfile for the host distro. - If not set, returns None. - """ - with open(SCHUTZFILE, encoding="utf-8") as schutzfile: - data = json.load(schutzfile) - - return data.get("common", {}).get("dependencies", {}).get("bootc-image-builder", {}).get("ref", None) - - -def rng_seed_env(): - """ - Read the rng seed from the Schutzfile and return it as a map to use as an environment variable with the appropriate - key. Assumes the file exists and that it contains the key 'rngseed', otherwise raises an exception. - """ - - with open(SCHUTZFILE, encoding="utf-8") as schutzfile: - data = json.load(schutzfile) - - seed = data.get("common", {}).get("rngseed") - if seed is None: - raise RuntimeError("'common.rngseed' not found in Schutzfile") - - return {"OSBUILD_TESTING_RNG_SEED": str(seed)} - - -def host_container_arch(): - host_arch = os.uname().machine - return { - "x86_64": "amd64", - "aarch64": "arm64" - }.get(host_arch, host_arch) - - -def is_manifest_list(data): - """Inspect a manifest determine if it's a multi-image manifest-list.""" - media_type = data.get("mediaType") - # Check if mediaType is set according to docker or oci specifications - if media_type in ("application/vnd.docker.distribution.manifest.list.v2+json", - "application/vnd.oci.image.index.v1+json"): - return True - - # According to the OCI spec, setting mediaType is not mandatory. So, if it is not set at all, check for the - # existence of manifests - if media_type is None and data.get("manifests") is not None: - return True - - return False - - -def skopeo_inspect_id(image_name: str, arch: str) -> str: - """ - Returns the image ID (config digest) of the container image. If the image resolves to a manifest list, the config - digest of the given architecture is resolved. - - Runs with 'sudo' when inspecting a local container because in our tests we need to read the root container storage. - """ - cmd = ["skopeo", "inspect", "--raw", image_name] - if image_name.startswith("containers-storage"): - cmd = ["sudo"] + cmd - out, _ = runcmd(cmd) - data = json.loads(out) - if not is_manifest_list(data): - return data["config"]["digest"] - - for manifest in data.get("manifests", []): - platform = manifest.get("platform", {}) - img_arch = platform.get("architecture", "") - img_ostype = platform.get("os", "") - - if arch != img_arch or img_ostype != "linux": - continue - - if "@" in image_name: - image_no_tag = image_name.split("@")[0] - else: - image_no_tag = ":".join(image_name.split(":")[:-1]) - manifest_digest = manifest["digest"] - arch_image_name = f"{image_no_tag}@{manifest_digest}" - # inspect the arch-specific manifest to get the image ID (config digest) - return skopeo_inspect_id(arch_image_name, arch) - - # don't error out, just return an empty string and let the caller handle it - return "" - - -def get_tag_for(runner): - if runner.startswith("aws/"): - return "terraform" - if runner.startswith("rhos-01/"): - return "terraform/openstack" - - raise ValueError(f"Unknown runner: {runner}") - - -def get_ci_runner_for(arch, image_type): - with open(SCHUTZFILE, encoding="utf-8") as schutzfile: - data = json.load(schutzfile) - - if (runner := data.get("common", {}).get("gitlab-ci-runner-for", {}).get(arch, {}).get(image_type)) is not None: - return runner - - return get_common_ci_runner() - - -def get_common_ci_runner(): - """ - CI runner for common tasks. - - Currently this is used for all gitlab CI jobs. In the future, we might switch to running build jobs on the same host - distro as the target image, but this CI runner will still be used for generic tasks like check-build-coverage. - """ - with open(SCHUTZFILE, encoding="utf-8") as schutzfile: - data = json.load(schutzfile) - - if (runner := data.get("common", {}).get("gitlab-ci-runner")) is None: - raise KeyError(f"gitlab-ci-runner not defined in {SCHUTZFILE}") - - return runner - - -def get_common_ci_runner_distro(): - """ - CI runner distro for common tasks. - - Returns the distro part from the value of the common.gitlab-ci-runner key in the Schutzfile. - For example, if the value is "aws/fedora-999", this function will return "fedora-999". - """ - return get_common_ci_runner().split("/")[1] - - -def find_image_file(build_path: str) -> str: - """ - Find the path to the image by reading the manifest and finding the exported pipeline's output directory. - A manifest may contain multiple pipelines but only one is exported during a build. This function finds the - exported pipeline by checking which pipeline directory exists in the build output. - Raises RuntimeError if no or multiple exported directories are found, or if the directory doesn't contain - exactly one file. - """ - manifest_file = os.path.join(build_path, "manifest.json") - with open(manifest_file, encoding="utf-8") as manifest: - data = json.load(manifest) - - pipeline_names = [p["name"] for p in data["pipelines"] if p["name"] != "build"] - export_dirs = [p for p in pipeline_names if os.path.isdir(os.path.join(build_path, p))] - - if len(export_dirs) != 1: - raise RuntimeError(f"Expected exactly one exported pipeline directory in {build_path}, found: {export_dirs}") - - files = os.listdir(os.path.join(build_path, export_dirs[0])) - if len(files) != 1: - raise RuntimeError( - f"Expected exactly one file in export directory '{export_dirs[0]}', found: {files}") - - return os.path.join(build_path, export_dirs[0], files[0]) - - -def read_build_info(build_path: str) -> Dict: - """ - Read the info.json file from the build directory and return the data as a dictionary. - """ - info_file_path = os.path.join(build_path, "info.json") - with open(info_file_path, encoding="utf-8") as info_fp: - return json.load(info_fp) - - -def read_manifest(build_path: str) -> Dict: - """ - Read the manifest.json file from the build directory and return the data as a dictionary. - """ - info_file_path = os.path.join(build_path, "manifest.json") - with open(info_file_path, encoding="utf-8") as info_fp: - return json.load(info_fp) - - -def write_build_info(build_path: str, data: Dict): - """ - Write the data to the info.json file in the build directory. - """ - info_file_path = os.path.join(build_path, "info.json") - with open(info_file_path, "w", encoding="utf-8") as info_fp: - json.dump(data, info_fp, indent=2) - - -def touch_s3(distro, arch, manifest_id, osbuild_ref=None, runner_distro=None): - """ - Update the timestamps of a path in S3 by adding a metadata field to each file recursively. This can be used to - "freshen up" relevant files in the build cache so that images that are still current but haven't been updated in a - while don't get garbage collected. - """ - s3url = gen_build_info_s3_dir_path(distro, arch, manifest_id, osbuild_ref, runner_distro) - # the exact key and value don't matter, but let's add the current datetime to make it a bit more meaningful - now = str(datetime.now()) - print(f"⌚ Updating timestamps for {s3url} ({now})") - cmd = ["aws", "s3", "cp", "--recursive", "--metadata", f"touched={now}", s3url, s3url] - runcmd_nc(cmd) - - -def running_in_gitlab(): - """ - Returns true if running in GitLab CI. - """ - return os.environ.get("GITLAB_CI") - - -def print_section_start(name: str, msg: str = ""): - """ - Prints a section header with a timestamp for logging output during tests. - If running in GitLab CI, it also creates a collapsible section. - - https://docs.gitlab.com/ci/jobs/job_logs/#custom-collapsible-sections - """ - now = datetime.now() - if running_in_gitlab(): - print(f"\033[0Ksection_start:{int(now.timestamp())}:{name}[collapsed=true]\r\033[0K{msg}") - return - - # custom line for non CI runs - isonow = now.isoformat() - print(f":: [{isonow}] {msg} ({name})") - - -def print_section_end(name: str): - now = datetime.now() - if running_in_gitlab(): - print(f"\033[0Ksection_end:{int(now.timestamp())}:{name}\r\033[0K") - return - - # custom line for non CI runs - isonow = now.isoformat() - print(f":: [{isonow}] Done ({name})") diff --git a/test/scripts/imgtestlib/run.py b/test/scripts/imgtestlib/run.py new file mode 100644 index 0000000000..afb50bb3f5 --- /dev/null +++ b/test/scripts/imgtestlib/run.py @@ -0,0 +1,33 @@ +import os +import subprocess as sp +import sys + + +def runcmd(cmd, stdin=None, extra_env=None, capture_output=True): + """ + Run the cmd using sp.run() and exit with the returncode if it is non-zero. + Output is captured and both stdout and stderr are returned if the run succeeds. + If it fails, the output is printed before exiting. + """ + env = None + if extra_env: + env = os.environ + env.update(extra_env) + job = sp.run(cmd, input=stdin, capture_output=capture_output, env=env, check=False) + if job.returncode > 0: + print(f"❌ Command failed: {cmd}") + if job.stdout: + print(job.stdout.decode()) + if job.stderr: + print(job.stderr.decode()) + sys.exit(job.returncode) + + return job.stdout, job.stderr + + +def runcmd_nc(cmd, stdin=None, extra_env=None): + """ + Run the cmd using sp.run() and exit with the returncode if it is non-zero. + Output it not captured. + """ + runcmd(cmd, stdin=stdin, extra_env=extra_env, capture_output=False) diff --git a/test/scripts/imgtestlib/testenv.py b/test/scripts/imgtestlib/testenv.py new file mode 100644 index 0000000000..54ebf3c336 --- /dev/null +++ b/test/scripts/imgtestlib/testenv.py @@ -0,0 +1,129 @@ +import json +import os +import pathlib + +# Path to the Schutzfile relative to the root of the repository +SCHUTZFILE = str(pathlib.Path(__file__).resolve().parents[3] / "Schutzfile") +OS_RELEASE_FILE = "/etc/os-release" + + +def get_host_distro(): + """ + Get the host distro version based on data in the os-release file. + The format is - (e.g. fedora-41). + + Can be overridden by setting the OSBUILD_IMGTESTLIB_HOST_DISTRO env var. + """ + # overriding this is useful for running tests locally on any distro version while still being able to reuse the + # cached images from the CI runners + if distro := os.environ.get("OSBUILD_IMGTESTLIB_HOST_DISTRO"): + return distro + osrelease = read_osrelease() + return f"{osrelease['ID']}-{osrelease['VERSION_ID']}" + + +def get_osbuild_commit(distro_version): + """ + Get the osbuild commit defined in the Schutzfile for the host distro or common. + If not set, returns None. + """ + with open(SCHUTZFILE, encoding="utf-8") as schutzfile: + data = json.load(schutzfile) + + commit = data.get(distro_version, {}).get("dependencies", {}).get("osbuild", {}).get("commit", None) + if commit is None: + commit = data.get("common", {}).get("dependencies", {}).get("osbuild", {}).get("commit", None) + return commit + + +def get_bib_ref(): + """ + Get the bootc-image-builder ref defined in the Schutzfile for the host distro. + If not set, returns None. + """ + with open(SCHUTZFILE, encoding="utf-8") as schutzfile: + data = json.load(schutzfile) + + return data.get("common", {}).get("dependencies", {}).get("bootc-image-builder", {}).get("ref", None) + + +def rng_seed_env(): + """ + Read the rng seed from the Schutzfile and return it as a map to use as an environment variable with the appropriate + key. Assumes the file exists and that it contains the key 'rngseed', otherwise raises an exception. + """ + + with open(SCHUTZFILE, encoding="utf-8") as schutzfile: + data = json.load(schutzfile) + + seed = data.get("common", {}).get("rngseed") + if seed is None: + raise RuntimeError("'common.rngseed' not found in Schutzfile") + + return {"OSBUILD_TESTING_RNG_SEED": str(seed)} + + +def read_osrelease(): + """Read Operating System Information from `os-release` + + This creates a dictionary with information describing the running operating system. It reads the information from + the path array provided as `paths`. The first available file takes precedence. It must be formatted according to + the rules in `os-release(5)`. + """ + osrelease = {} + + with open(OS_RELEASE_FILE, encoding="utf8") as orf: + for line in orf: + line = line.strip() + if not line: + continue + if line[0] == "#": + continue + key, value = line.split("=", 1) + osrelease[key] = value.strip('"') + + return osrelease + + +def host_container_arch(): + host_arch = os.uname().machine + return { + "x86_64": "amd64", + "aarch64": "arm64" + }.get(host_arch, host_arch) + + +def get_ci_runner_for(arch, image_type): + with open(SCHUTZFILE, encoding="utf-8") as schutzfile: + data = json.load(schutzfile) + + if (runner := data.get("common", {}).get("gitlab-ci-runner-for", {}).get(arch, {}).get(image_type)) is not None: + return runner + + return get_common_ci_runner() + + +def get_common_ci_runner(): + """ + CI runner for common tasks. + + Currently this is used for all gitlab CI jobs. In the future, we might switch to running build jobs on the same host + distro as the target image, but this CI runner will still be used for generic tasks like check-build-coverage. + """ + with open(SCHUTZFILE, encoding="utf-8") as schutzfile: + data = json.load(schutzfile) + + if (runner := data.get("common", {}).get("gitlab-ci-runner")) is None: + raise KeyError(f"gitlab-ci-runner not defined in {SCHUTZFILE}") + + return runner + + +def get_common_ci_runner_distro(): + """ + CI runner distro for common tasks. + + Returns the distro part from the value of the common.gitlab-ci-runner key in the Schutzfile. + For example, if the value is "aws/fedora-999", this function will return "fedora-999". + """ + return get_common_ci_runner().split("/")[1] diff --git a/test/scripts/test_imgtestlib.py b/test/scripts/test_imgtestlib.py index 61232a182a..477eaab4d3 100644 --- a/test/scripts/test_imgtestlib.py +++ b/test/scripts/test_imgtestlib.py @@ -114,8 +114,9 @@ def test_read_seed(): ), )) def test_gen_build_info_dir_path_prefix(kwargs, expected): - with patch("imgtestlib.imgtestlib.get_host_distro", return_value="fedora-999"), \ - patch("imgtestlib.imgtestlib.get_osbuild_commit", return_value="abcdef123456"): + # we need to patch the functions that were imported into the cache namespace, not the originals in .testenv + with patch("imgtestlib.cache.get_host_distro", return_value="fedora-999"), \ + patch("imgtestlib.cache.get_osbuild_commit", return_value="abcdef123456"): assert testlib.gen_build_info_dir_path_prefix(**kwargs) == expected @@ -128,8 +129,8 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "arch": "aarch64", "manifest_id": "abc123" }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + \ - "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/aarch64/manifest-id-abc123/", + testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/aarch64/manifest-id-abc123/", ), ( { @@ -138,8 +139,8 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "distro": "fedora-41", "arch": "aarch64", }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + \ - "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/aarch64/", + testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/aarch64/", ), ( { @@ -147,16 +148,16 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "runner_distro": "fedora-41", "distro": "fedora-41", }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + \ - "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/", + testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/", ), ( { "osbuild_ref": "abcdef123456", "runner_distro": "fedora-41", }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + \ - "/osbuild-ref-abcdef123456/runner-fedora-41/", + testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + "/osbuild-ref-abcdef123456/runner-fedora-41/", ), # Optional arg 'distro' not specified, thus following optional args 'arch' and 'manifest_id' are ignored ( @@ -166,8 +167,8 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "arch": "aarch64", "manifest_id": "abc123" }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + \ - "/osbuild-ref-abcdef123456/runner-fedora-41/", + testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + "/osbuild-ref-abcdef123456/runner-fedora-41/", ), # Optional arg 'arch' not specified, thus following optional arg 'manifest_id' is ignored ( @@ -177,8 +178,8 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): "distro": "fedora-41", "manifest_id": "abc123" }, - testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + \ - "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/", + testlib.S3_BUCKET + "/" + testlib.S3_PREFIX + + "/osbuild-ref-abcdef123456/runner-fedora-41/fedora-41/", ), # default osbuild_ref ( @@ -201,8 +202,9 @@ def test_gen_build_info_dir_path_prefix(kwargs, expected): ), )) def test_gen_build_info_s3_dir_path(kwargs, expected): - with patch("imgtestlib.imgtestlib.get_host_distro", return_value="fedora-999"), \ - patch("imgtestlib.imgtestlib.get_osbuild_commit", return_value="abcdef123456"): + # we need to patch the functions that were imported into the cache namespace, not the originals in .testenv + with patch("imgtestlib.cache.get_host_distro", return_value="fedora-999"), \ + patch("imgtestlib.cache.get_osbuild_commit", return_value="abcdef123456"): assert testlib.gen_build_info_s3_dir_path(**kwargs) == expected From c801842bf106ab433aa870d6e2f84961614a3e2a Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 21 May 2026 14:42:14 +0200 Subject: [PATCH 06/15] test/imgtestlib: add a red box on cache misses Make them easier to see in the log. --- test/scripts/imgtestlib/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scripts/imgtestlib/core.py b/test/scripts/imgtestlib/core.py index 4db9c5e5ad..904848eb74 100644 --- a/test/scripts/imgtestlib/core.py +++ b/test/scripts/imgtestlib/core.py @@ -143,7 +143,7 @@ def check_for_build(manifest_fname, build_request, manifest_data, build_info_dir build_info_path = os.path.join(build_info_dir, "info.json") # rebuild if matching build info is not found if not os.path.exists(build_info_path): - print(f"Build info not found: {build_info_path}") + print(f"🟥 Build info not found: {build_info_path}") print(" Adding config to build pipeline.") return True From f0fcc8eb3a27f4b44b4aa52bea7ea537e7b5b632 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 21 May 2026 14:35:54 +0200 Subject: [PATCH 07/15] test: move core of boot-image script into library function The boot-image script is now a thin wrapper around the new imgtestlib.boot_image() function, so the general functionality is reusable. The boot-image script now simply handles argument parsing and calls into the function. The can_boot_test() function has been moved to imgtestlib/core to avoid circular imports. We might need to reorganise the module at a later date. --- test/scripts/boot-image | 70 +--------------- test/scripts/imgtestlib/boot.py | 140 +++++++++++++++++--------------- test/scripts/imgtestlib/core.py | 70 +++++++++++++++- 3 files changed, 144 insertions(+), 136 deletions(-) diff --git a/test/scripts/boot-image b/test/scripts/boot-image index 05130b860d..c39d32ae4c 100755 --- a/test/scripts/boot-image +++ b/test/scripts/boot-image @@ -1,13 +1,9 @@ #!/usr/bin/env python3 import argparse -import json -import os -import pathlib import imgtestlib as testlib -# pylint: disable=too-many-branches def main(): desc = "Boot an image in the cloud environment it is built for and validate the configuration" parser = argparse.ArgumentParser(description=desc) @@ -20,71 +16,9 @@ def main(): args = parser.parse_args() search_path = args.image_search_path build_config_path = args.config + keep = args.keep_booted - image_path = testlib.find_image_file(search_path) - build_info = testlib.read_build_info(search_path) - distro = build_info["distro"] - arch = build_info["arch"] - image_type = build_info["image-type"] - - # NOTE: Some installer ISOs have embedded kickstart, but they are interactive, so we need to embed - # a custom kickstart with non-interactive settings in it. To preserve the original kickstart content, - # we need to read the original kickstart content from the build directory and merge it with the custom - # kickstart content. - iso_embedded_ks = build_info.get("iso-embedded-ks", None) - iso_embedded_ks_path = None - if iso_embedded_ks: - iso_embedded_ks_path = os.path.join(search_path, iso_embedded_ks) - if not os.path.exists(iso_embedded_ks_path): - raise RuntimeError(f"'iso-embedded-ks' specified in the info.json, but file not found: {iso_embedded_ks_path}") - - config = json.loads(pathlib.Path(build_config_path).read_text(encoding="utf8")) - if not testlib.can_boot_test(image_type, testlib.read_manifest(search_path), - image_type, arch, distro, config.get("blueprint", {})): - print(f"SKIP: {image_type} boot tests on {arch} are not supported ({distro})") - return - - print(f"Testing image at {image_path}") - bib_image_id = "" - match image_type: - # Not all qcow2 types can be boot-tested, for example `server-qcow2` uses - # initial-setup and this blocks the boot. - case "qcow2" | "generic-qcow2" | "cloud-qcow2": - testlib.boot_qemu(arch, image_path, build_config_path, keep_booted=args.keep_booted) - case "image-installer" | "minimal-installer": - testlib.boot_qemu_iso(arch, image_path, build_config_path) - case "network-installer" | "everything-network-installer" | "bootc-generic-iso": - testlib.boot_qemu_iso_no_unattended_support(arch, image_path, build_config_path) - case "pxe-tar-xz": - testlib.boot_qemu_pxe(arch, image_path) - case "ami" | "ec2" | "ec2-ha" | "ec2-sap" | "edge-ami" | "cloud-ec2": - testlib.boot_ami(distro, arch, image_type, image_path, build_config_path) - case "vhd": - testlib.boot_vhd(distro, arch, image_path, build_config_path) - case "iot-bootable-container": - manifest_id = build_info["manifest-checksum"] - testlib.boot_container(distro, arch, image_type, image_path, manifest_id, build_config_path) - bib_ref = testlib.get_bib_ref() - bib_image_id = testlib.skopeo_inspect_id(f"docker://{bib_ref}", testlib.host_container_arch()) - case "wsl" | "generic-wsl": - if distro == "fedora-41": - print(f"{distro} {image_type} boot tests are not supported, fails on wsl import") - return - testlib.boot_wsl(distro, arch, image_path, build_config_path) - case _: - raise testlib.MissingBootImplementation(f"{arch} {image_type} is missing a boot implementation.") - - print("✅ Marking boot successful") - # amend build info with boot success - # search_path is the root of the build path (build/build_name) - build_info["boot-success"] = True - testlib.write_build_info(search_path, build_info) - if bib_image_id: - # write a separate file with the bib image ID as filename to mark the boot success with that image - bib_id_file = os.path.join(search_path, f"bib-{bib_image_id}") - print(f"Writing bib image ID file: {bib_id_file}") - with open(bib_id_file, "w", encoding="utf-8") as fp: - fp.write("") + testlib.boot_image(search_path, build_config_path, keep) if __name__ == "__main__": diff --git a/test/scripts/imgtestlib/boot.py b/test/scripts/imgtestlib/boot.py index e1c987a771..b687f9714c 100644 --- a/test/scripts/imgtestlib/boot.py +++ b/test/scripts/imgtestlib/boot.py @@ -15,8 +15,11 @@ from vmtest.util import get_free_port from vmtest.vm import QEMU +from .build import read_build_info, write_build_info +from .core import (can_boot_test, find_image_file, read_manifest, + skopeo_inspect_id) from .run import runcmd, runcmd_nc -from .testenv import get_bib_ref +from .testenv import get_bib_ref, host_container_arch BASE_TEST_EXEC = "check-host-config-" # + arch WSL_TEST_SCRIPT = "test/scripts/wsl-entrypoint.bat" @@ -26,26 +29,6 @@ REGISTRY = "registry.gitlab.com/redhat/services/products/image-builder/ci/images" -# image types that can be boot tested -# Keep in sync with test/scripts/boot-image which has the same checks again -CAN_BOOT_TEST = { - "*": [ - "ami", - "ec2", - "ec2-ha", - "ec2-sap", - "edge-ami", - "iot-bootable-container", - "vhd", - "cloud-ec2", - ], - "x86_64": [ - "image-installer", "minimal-installer", "network-installer", - "qcow2", "generic-qcow2", "cloud-qcow2", - "wsl", "generic-wsl", - ] -} - def get_aws_config(): return { @@ -537,49 +520,72 @@ def boot_wsl(distro, arch, image_path, config): runcmd_nc(cmd) -# pylint: disable=too-many-return-statements,too-many-arguments,too-many-positional-arguments -def can_boot_test(manifest_fname, manifest_data, image_type, arch, distro, blueprint): - if image_type not in CAN_BOOT_TEST.get("*", []) + CAN_BOOT_TEST.get(arch, []): - return False - - if image_type in ["image-installer", "minimal-installer"]: - if not blueprint.get("customizations", {}).get("installer", {}).get("unattended"): - print(" not bootable: only unattended installers are supported") - return False - - if image_type in ["network-installer", "everything-network-installer", "server-network-installer"]: - if distro in ["rhel-10.1", "rhel-10.2"]: - print(" not bootable: rhel network-installer tests have incomplete repos in nightly snapshot" - "and won't install") - return False - if distro.startswith("fedora"): - print(" not bootable: fedora network-installer crashes in sshd," - "see https://bugzilla.redhat.com/show_bug.cgi?id=2415883") - return False - if distro == "centos-9": - print(" not bootable: centos-9 will not start an install and waits on source selection") - return False - if distro.startswith("rhel-9"): - print(" not bootable: rhel-9 will not start an install and waits on source selection") - return False - - if image_type in ["qcow2", "generic-qcow2", "cloud-qcow2", "image-installer", "minimal-installer", - "network-installer", "everything-network-installer"]: - if blueprint.get("customizations", {}).get("fips") and distro.startswith("fedora"): - print(" not bootable: fips on fedora is unstable, fails with e.g. dracut:" - "FATAL: FIPS integrity test failed") - return False - # Note that this needs adjustment when we switch to librepo - urls = [src["url"] for src in manifest_data["sources"]["org.osbuild.curl"]["items"].values()] - if not any("ssh-server" in url for url in urls): - # This can happen e.g. when an image is build with the "minimal: true" customization. - # We could use guestfs to inject keys, see PR#1995 - print(f" not bootable: ssh-server not found in manifest {manifest_fname} ({arch} {image_type})") - return False - # We need jq in the image many images do not have it - # (e.g. centos-9/rhel-9 with releasever config) so skip those too - if not any("jq" in url for url in urls): - print(f" not bootable: jq not found in {manifest_fname} ({arch} {image_type})") - return False - - return True +# pylint: disable=too-many-branches +def boot_image(search_path, build_config_path, keep_booted=False): + image_path = find_image_file(search_path) + build_info = read_build_info(search_path) + distro = build_info["distro"] + arch = build_info["arch"] + image_type = build_info["image-type"] + + # NOTE: Some installer ISOs have embedded kickstart, but they are interactive, so we need to embed + # a custom kickstart with non-interactive settings in it. To preserve the original kickstart content, + # we need to read the original kickstart content from the build directory and merge it with the custom + # kickstart content. + iso_embedded_ks = build_info.get("iso-embedded-ks", None) + iso_embedded_ks_path = None + if iso_embedded_ks: + iso_embedded_ks_path = os.path.join(search_path, iso_embedded_ks) + if not os.path.exists(iso_embedded_ks_path): + raise RuntimeError( + f"'iso-embedded-ks' specified in the info.json, but file not found: {iso_embedded_ks_path}" + ) + + config = json.loads(pathlib.Path(build_config_path).read_text(encoding="utf8")) + manifest_path = os.path.join(search_path, "manifest.json") + if not can_boot_test(manifest_path, read_manifest(search_path), + image_type, arch, distro, config.get("blueprint", {})): + print(f"SKIP: {image_type} boot tests on {arch} are not supported ({distro})") + return + + print(f"Testing image at {image_path}") + bib_image_id = "" + match image_type: + # Not all qcow2 types can be boot-tested, for example `server-qcow2` uses + # initial-setup and this blocks the boot. + case "qcow2" | "generic-qcow2" | "cloud-qcow2": + boot_qemu(arch, image_path, build_config_path, keep_booted=keep_booted) + case "image-installer" | "minimal-installer": + boot_qemu_iso(arch, image_path, build_config_path) + case "network-installer" | "everything-network-installer" | "bootc-generic-iso": + boot_qemu_iso_no_unattended_support(arch, image_path, build_config_path) + case "pxe-tar-xz": + boot_qemu_pxe(arch, image_path) + case "ami" | "ec2" | "ec2-ha" | "ec2-sap" | "edge-ami" | "cloud-ec2": + boot_ami(distro, arch, image_type, image_path, build_config_path) + case "vhd": + boot_vhd(distro, arch, image_path, build_config_path) + case "iot-bootable-container": + manifest_id = build_info["manifest-checksum"] + boot_container(distro, arch, image_type, image_path, manifest_id, build_config_path) + bib_ref = get_bib_ref() + bib_image_id = skopeo_inspect_id(f"docker://{bib_ref}", host_container_arch()) + case "wsl" | "generic-wsl": + if distro == "fedora-41": + print(f"{distro} {image_type} boot tests are not supported, fails on wsl import") + return + boot_wsl(distro, arch, image_path, build_config_path) + case _: + raise MissingBootImplementation(f"{arch} {image_type} is missing a boot implementation.") + + print("✅ Marking boot successful") + # amend build info with boot success + # search_path is the root of the build path (build/build_name) + build_info["boot-success"] = True + write_build_info(search_path, build_info) + if bib_image_id: + # write a separate file with the bib image ID as filename to mark the boot success with that image + bib_id_file = os.path.join(search_path, f"bib-{bib_image_id}") + print(f"Writing bib image ID file: {bib_id_file}") + with open(bib_id_file, "w", encoding="utf-8") as fp: + fp.write("") diff --git a/test/scripts/imgtestlib/core.py b/test/scripts/imgtestlib/core.py index 904848eb74..d5a09666dc 100644 --- a/test/scripts/imgtestlib/core.py +++ b/test/scripts/imgtestlib/core.py @@ -6,7 +6,6 @@ from glob import glob from typing import Dict -from .boot import can_boot_test from .build import get_manifest_id from .cache import dl_build_info, gen_build_info_dir_path_prefix, touch_s3 from .gitlab import print_section_end, print_section_start @@ -22,6 +21,27 @@ ] +# image types that can be boot tested +# Keep in sync with test/scripts/boot-image which has the same checks again +CAN_BOOT_TEST = { + "*": [ + "ami", + "ec2", + "ec2-ha", + "ec2-sap", + "edge-ami", + "iot-bootable-container", + "vhd", + "cloud-ec2", + ], + "x86_64": [ + "image-installer", "minimal-installer", "network-installer", + "qcow2", "generic-qcow2", "cloud-qcow2", + "wsl", "generic-wsl", + ] +} + + # base and terraform bits copied from main .gitlab-ci.yml # needed for status reporting and defining the runners BASE_CONFIG = """ @@ -369,3 +389,51 @@ def read_manifest(build_path: str) -> Dict: info_file_path = os.path.join(build_path, "manifest.json") with open(info_file_path, encoding="utf-8") as info_fp: return json.load(info_fp) + + +# pylint: disable=too-many-return-statements,too-many-arguments,too-many-positional-arguments +def can_boot_test(manifest_fname, manifest_data, image_type, arch, distro, blueprint): + if image_type not in CAN_BOOT_TEST.get("*", []) + CAN_BOOT_TEST.get(arch, []): + return False + + if image_type in ["image-installer", "minimal-installer"]: + if not blueprint.get("customizations", {}).get("installer", {}).get("unattended"): + print(" not bootable: only unattended installers are supported") + return False + + if image_type in ["network-installer", "everything-network-installer", "server-network-installer"]: + if distro in ["rhel-10.1", "rhel-10.2"]: + print(" not bootable: rhel network-installer tests have incomplete repos in nightly snapshot" + "and won't install") + return False + if distro.startswith("fedora"): + print(" not bootable: fedora network-installer crashes in sshd," + "see https://bugzilla.redhat.com/show_bug.cgi?id=2415883") + return False + if distro == "centos-9": + print(" not bootable: centos-9 will not start an install and waits on source selection") + return False + if distro.startswith("rhel-9"): + print(" not bootable: rhel-9 will not start an install and waits on source selection") + return False + + if image_type in ["qcow2", "generic-qcow2", "cloud-qcow2", "image-installer", "minimal-installer", + "network-installer", "everything-network-installer"]: + if blueprint.get("customizations", {}).get("fips") and distro.startswith("fedora"): + print(" not bootable: fips on fedora is unstable, fails with e.g. dracut:" + "FATAL: FIPS integrity test failed") + return False + # Note that this needs adjustment when we switch to librepo + urls = [src["url"] for src in manifest_data["sources"]["org.osbuild.curl"]["items"].values()] + if not any("ssh-server" in url for url in urls): + # This can happen e.g. when an image is build with the "minimal: true" customization. + # We could use guestfs to inject keys, see PR#1995 + print(f" not bootable: ssh-server not found in manifest {manifest_fname} ({arch} {image_type})") + return False + # We need jq in the image many images do not have it + # (e.g. centos-9/rhel-9 with releasever config) so skip those too + if not any("jq" in url for url in urls): + print(f" not bootable: jq not found in {manifest_fname} ({arch} {image_type})") + return False + + return True From 313323552aca1d9c6fa45d9e91df1477f3dfe166 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 21 May 2026 16:27:47 +0200 Subject: [PATCH 08/15] test: move core of upload-results script into library function The upload-results script is now a thin wrapper around the new imgtestlib.upload_results() function, so the general functionality is reusable. The upload-results script now simply handles argument parsing and calls into the function. --- test/scripts/imgtestlib/cache.py | 37 ++++++++++++++++++++++++++++++++ test/scripts/upload-results | 33 +--------------------------- 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/test/scripts/imgtestlib/cache.py b/test/scripts/imgtestlib/cache.py index 8093770bb9..e83db0eed5 100644 --- a/test/scripts/imgtestlib/cache.py +++ b/test/scripts/imgtestlib/cache.py @@ -1,8 +1,11 @@ +import json import os import subprocess as sp from datetime import datetime from typing import List, Optional +from .build import (gen_build_name, get_manifest_id, read_build_info, + write_build_info) from .run import runcmd_nc from .testenv import get_host_distro, get_osbuild_commit @@ -110,3 +113,37 @@ def touch_s3(distro, arch, manifest_id, osbuild_ref=None, runner_distro=None): print(f"⌚ Updating timestamps for {s3url} ({now})") cmd = ["aws", "s3", "cp", "--recursive", "--metadata", f"touched={now}", s3url, s3url] runcmd_nc(cmd) + + +def upload_results(distro, arch, image_type, config_path): + with open(config_path, "r", encoding="utf-8") as config_file: + config = json.load(config_file) + config_name = config["name"] + + build_dir = os.path.join("build", gen_build_name(distro, arch, image_type, config_name)) + + # get the manifest ID to use in the destination path + manifest_path = os.path.join(build_dir, "manifest.json") + with open(manifest_path, "r", encoding="utf-8") as manifest_fp: + manifest_data = json.load(manifest_fp) + manifest_id = get_manifest_id(manifest_data) + + # add the PR number (gitlab branch name) to the info.json if available + if pr_number := os.environ.get("CI_COMMIT_BRANCH"): + build_info = read_build_info(build_dir) + # strip the PR prefix + build_info["pr"] = pr_number.removeprefix("PR-") + write_build_info(build_dir, build_info) + + s3url = gen_build_info_s3_dir_path(distro, arch, manifest_id) + + # It can happen that the upload fails before finishing. This can cause problems with inconsistent cache state if, + # for example, the info.json file is uploaded but the manifest or image is not. Since the info.json is the important + # file for identifying if a build was successful, let's upload everything else first, without info.json, and then + # upload the info.json separately as a final step. + print(f"⬆️ Uploading {build_dir} to {s3url} (without info)") + runcmd_nc(["aws", "s3", "cp", "--no-progress", "--acl=private", "--recursive", "--exclude=info.json", + build_dir+"/", s3url]) + print(f"⬆️ Uploading info.json to {s3url}") + runcmd_nc(["aws", "s3", "cp", "--no-progress", "--acl=private", "--recursive", build_dir+"/", s3url]) + print("✅ DONE!!") diff --git a/test/scripts/upload-results b/test/scripts/upload-results index 39b6ad9b6b..4eae11d43e 100755 --- a/test/scripts/upload-results +++ b/test/scripts/upload-results @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import argparse -import json import os import imgtestlib as testlib @@ -19,37 +18,7 @@ def main(): config_path = args.config arch = os.uname().machine - with open(config_path, "r", encoding="utf-8") as config_file: - config = json.load(config_file) - config_name = config["name"] - - build_dir = os.path.join("build", testlib.gen_build_name(distro, arch, image_type, config_name)) - - # get the manifest ID to use in the destination path - manifest_path = os.path.join(build_dir, "manifest.json") - with open(manifest_path, "r", encoding="utf-8") as manifest_fp: - manifest_data = json.load(manifest_fp) - manifest_id = testlib.get_manifest_id(manifest_data) - - # add the PR number (gitlab branch name) to the info.json if available - if pr_number := os.environ.get("CI_COMMIT_BRANCH"): - build_info = testlib.read_build_info(build_dir) - # strip the PR prefix - build_info["pr"] = pr_number.removeprefix("PR-") - testlib.write_build_info(build_dir, build_info) - - s3url = testlib.gen_build_info_s3_dir_path(distro, arch, manifest_id) - - # It can happen that the upload fails before finishing. This can cause problems with inconsistent cache state if, - # for example, the info.json file is uploaded but the manifest or image is not. Since the info.json is the important - # file for identifying if a build was successful, let's upload everything else first, without info.json, and then - # upload the info.json separately as a final step. - print(f"⬆️ Uploading {build_dir} to {s3url} (without info)") - testlib.runcmd_nc(["aws", "s3", "cp", "--no-progress", "--acl=private", "--recursive", "--exclude=info.json", - build_dir+"/", s3url]) - print(f"⬆️ Uploading info.json to {s3url}") - testlib.runcmd_nc(["aws", "s3", "cp", "--no-progress", "--acl=private", "--recursive", build_dir+"/", s3url]) - print("✅ DONE!!") + testlib.upload_results(distro, arch, image_type, config_path) if __name__ == "__main__": From 51cc00c1655d359e001b9f7076a6d9a7964b3d23 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 21 May 2026 17:01:48 +0200 Subject: [PATCH 09/15] Schutzfile: bump the rngseed Let's test everything! --- Schutzfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Schutzfile b/Schutzfile index 77d39362d1..2fc9da0d7a 100644 --- a/Schutzfile +++ b/Schutzfile @@ -1,6 +1,6 @@ { "common": { - "rngseed": 2026051900, + "rngseed": 2026052100, "dependencies": { "bootc-image-builder": { "ref": "quay.io/centos-bootc/bootc-image-builder@sha256:9893e7209e5f449b86ababfd2ee02a58cca2e5990f77b06c3539227531fc8120" From 7231115220420544d50095b5314603666b58f95a Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 21 May 2026 17:18:43 +0200 Subject: [PATCH 10/15] test/vmtest: allow importing the module without boto3 boto3 is only needed when interfacing with aws. vmtest is imported by imgtestlib, which we often use for a lot of other smaller tasks, like setting up the osbuild repo. We could install boto3 whenever we need it, but it's simpler to allow importing without it. --- test/scripts/vmtest/vm.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/scripts/vmtest/vm.py b/test/scripts/vmtest/vm.py index a4dad84d9f..5c3a2eb88a 100644 --- a/test/scripts/vmtest/vm.py +++ b/test/scripts/vmtest/vm.py @@ -11,8 +11,15 @@ import uuid from io import StringIO -import boto3 -from botocore.exceptions import ClientError +try: + # The vmtest module is imported by imgtestlib. + # Allow importing the module without boto3 since most times it's not needed. + import boto3 + from botocore.exceptions import ClientError +except ImportError: + boto3 = None + ClientError = None + from vmtest.util import get_free_port, wait_ssh_ready AWS_REGION = "us-east-1" From 87fbc511b6f5a13507d2fb6e8c96e279fc3e3f82 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 21 May 2026 17:42:05 +0200 Subject: [PATCH 11/15] test/imgtestlib: replace match statement with if-chain Since we moved core parts of the test scripts into the imgtestlib module, the boot_image function now needs to be importable by all the distros we support and test on, including EL9 which only has Python 3.9. This wasn't an issue before because boot-image was only ever run on the CI runners, which are currently Fedora 42. Now that the core functionality is part of the importable module though, we need to rewrite it to run on older Python versions. --- test/scripts/imgtestlib/boot.py | 49 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/test/scripts/imgtestlib/boot.py b/test/scripts/imgtestlib/boot.py index b687f9714c..8efed51cda 100644 --- a/test/scripts/imgtestlib/boot.py +++ b/test/scripts/imgtestlib/boot.py @@ -550,33 +550,32 @@ def boot_image(search_path, build_config_path, keep_booted=False): print(f"Testing image at {image_path}") bib_image_id = "" - match image_type: + if image_type in ("qcow2", "generic-qcow2", "cloud-qcow2"): # Not all qcow2 types can be boot-tested, for example `server-qcow2` uses # initial-setup and this blocks the boot. - case "qcow2" | "generic-qcow2" | "cloud-qcow2": - boot_qemu(arch, image_path, build_config_path, keep_booted=keep_booted) - case "image-installer" | "minimal-installer": - boot_qemu_iso(arch, image_path, build_config_path) - case "network-installer" | "everything-network-installer" | "bootc-generic-iso": - boot_qemu_iso_no_unattended_support(arch, image_path, build_config_path) - case "pxe-tar-xz": - boot_qemu_pxe(arch, image_path) - case "ami" | "ec2" | "ec2-ha" | "ec2-sap" | "edge-ami" | "cloud-ec2": - boot_ami(distro, arch, image_type, image_path, build_config_path) - case "vhd": - boot_vhd(distro, arch, image_path, build_config_path) - case "iot-bootable-container": - manifest_id = build_info["manifest-checksum"] - boot_container(distro, arch, image_type, image_path, manifest_id, build_config_path) - bib_ref = get_bib_ref() - bib_image_id = skopeo_inspect_id(f"docker://{bib_ref}", host_container_arch()) - case "wsl" | "generic-wsl": - if distro == "fedora-41": - print(f"{distro} {image_type} boot tests are not supported, fails on wsl import") - return - boot_wsl(distro, arch, image_path, build_config_path) - case _: - raise MissingBootImplementation(f"{arch} {image_type} is missing a boot implementation.") + boot_qemu(arch, image_path, build_config_path, keep_booted=keep_booted) + elif image_type in ("image-installer", "minimal-installer"): + boot_qemu_iso(arch, image_path, build_config_path) + elif image_type in ("network-installer", "everything-network-installer", "bootc-generic-iso"): + boot_qemu_iso_no_unattended_support(arch, image_path, build_config_path) + elif image_type in ("pxe-tar-xz"): + boot_qemu_pxe(arch, image_path) + elif image_type in ("ami", "ec2", "ec2-ha", "ec2-sap", "edge-ami", "cloud-ec2"): + boot_ami(distro, arch, image_type, image_path, build_config_path) + elif image_type in ("vhd"): + boot_vhd(distro, arch, image_path, build_config_path) + elif image_type in ("iot-bootable-container"): + manifest_id = build_info["manifest-checksum"] + boot_container(distro, arch, image_type, image_path, manifest_id, build_config_path) + bib_ref = get_bib_ref() + bib_image_id = skopeo_inspect_id(f"docker://{bib_ref}", host_container_arch()) + elif image_type in ("wsl", "generic-wsl"): + if distro == "fedora-41": + print(f"{distro} {image_type} boot tests are not supported, fails on wsl import") + return + boot_wsl(distro, arch, image_path, build_config_path) + else: + raise MissingBootImplementation(f"{arch} {image_type} is missing a boot implementation.") print("✅ Marking boot successful") # amend build info with boot success From a7be7e50b30901cd890b93bc42182039492dc0d3 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 28 May 2026 15:22:36 +0200 Subject: [PATCH 12/15] test/imgtestlib: create a log_section() context decorator Support decorating functions with a log section context. This creates a log section at the start of the function that exits automatically at the end of the function. --- test/scripts/imgtestlib/gitlab.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/scripts/imgtestlib/gitlab.py b/test/scripts/imgtestlib/gitlab.py index b911123b23..903501d299 100644 --- a/test/scripts/imgtestlib/gitlab.py +++ b/test/scripts/imgtestlib/gitlab.py @@ -1,4 +1,6 @@ +import contextlib import os +import uuid from datetime import datetime @@ -35,3 +37,16 @@ def print_section_end(name: str): # custom line for non CI runs isonow = now.isoformat() print(f":: [{isonow}] Done ({name})") + + +class log_section(contextlib.ContextDecorator): + + def __init__(self, message): + self._id = str(uuid.uuid4()) + self._message = message + + def __enter__(self): + print_section_start(self._id, self._message) + + def __exit__(self, *_): + print_section_end(self._id) From a95de2e318948cd3639ae28968aa05244fa0562b Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 28 May 2026 16:28:03 +0200 Subject: [PATCH 13/15] test: use log_section decorator to manage log sections Use the log_section decorator to print the section start and end lines when entering and exiting certain functions. This makes the names of log sections static, where previously we could print specific values like the name or configuration of the image being built. While it would be nice to have this kind of information in the names of sections, I think the decorator solution is cleaner in code and the downside of having static log section names overall isn't too bad. --- test/scripts/generate-build-config | 1 + test/scripts/imgtestlib/boot.py | 2 ++ test/scripts/imgtestlib/build.py | 7 +++---- test/scripts/imgtestlib/cache.py | 2 ++ test/scripts/imgtestlib/core.py | 6 +++--- test/scripts/setup-osbuild-repo | 1 + 6 files changed, 12 insertions(+), 7 deletions(-) diff --git a/test/scripts/generate-build-config b/test/scripts/generate-build-config index 7b9dc63c0f..6086b99ac4 100755 --- a/test/scripts/generate-build-config +++ b/test/scripts/generate-build-config @@ -25,6 +25,7 @@ build/{distro}/{arch}/{image_type}/{config_name}: """ +@testlib.log_section("Generating manifests") def generate_manifests(outputdir, distro, arch): """ Generate all manifest using the default config list and return a dictionary mapping each manifest file to the diff --git a/test/scripts/imgtestlib/boot.py b/test/scripts/imgtestlib/boot.py index 8efed51cda..1a0099627b 100644 --- a/test/scripts/imgtestlib/boot.py +++ b/test/scripts/imgtestlib/boot.py @@ -18,6 +18,7 @@ from .build import read_build_info, write_build_info from .core import (can_boot_test, find_image_file, read_manifest, skopeo_inspect_id) +from .gitlab import log_section from .run import runcmd, runcmd_nc from .testenv import get_bib_ref, host_container_arch @@ -521,6 +522,7 @@ def boot_wsl(distro, arch, image_path, config): # pylint: disable=too-many-branches +@log_section("Booting image") def boot_image(search_path, build_config_path, keep_booted=False): image_path = find_image_file(search_path) build_info = read_build_info(search_path) diff --git a/test/scripts/imgtestlib/build.py b/test/scripts/imgtestlib/build.py index 4ab143a636..b9d65304e8 100644 --- a/test/scripts/imgtestlib/build.py +++ b/test/scripts/imgtestlib/build.py @@ -2,19 +2,19 @@ import os from typing import Dict -from .gitlab import print_section_end, print_section_start +from .gitlab import log_section from .run import runcmd, runcmd_nc from .testenv import get_host_distro, get_osbuild_commit, rng_seed_env +@log_section("Building image") def build_image(distro, arch, image_type, config_path): with open(config_path, "r", encoding="utf-8") as config_file: config = json.load(config_file) config_name = config["name"] - log_section_name = f"build_{distro}_{image_type}_{config_name}" - print_section_start(log_section_name, f"👷 Building image {distro}/{image_type} using config {config_path}") + print(f"👷 Building image {distro}/{image_type} using config {config_path}") # print the config for logging print(json.dumps(config, indent=2)) @@ -55,7 +55,6 @@ def build_image(distro, arch, image_type, config_path): "runner-distro": distro_version, } write_build_info(build_dir, build_info) - print_section_end(log_section_name) def read_build_info(build_path: str) -> Dict: diff --git a/test/scripts/imgtestlib/cache.py b/test/scripts/imgtestlib/cache.py index e83db0eed5..20c69386fe 100644 --- a/test/scripts/imgtestlib/cache.py +++ b/test/scripts/imgtestlib/cache.py @@ -6,6 +6,7 @@ from .build import (gen_build_name, get_manifest_id, read_build_info, write_build_info) +from .gitlab import log_section from .run import runcmd_nc from .testenv import get_host_distro, get_osbuild_commit @@ -115,6 +116,7 @@ def touch_s3(distro, arch, manifest_id, osbuild_ref=None, runner_distro=None): runcmd_nc(cmd) +@log_section("Uploading results") def upload_results(distro, arch, image_type, config_path): with open(config_path, "r", encoding="utf-8") as config_file: config = json.load(config_file) diff --git a/test/scripts/imgtestlib/core.py b/test/scripts/imgtestlib/core.py index d5a09666dc..380465316a 100644 --- a/test/scripts/imgtestlib/core.py +++ b/test/scripts/imgtestlib/core.py @@ -8,7 +8,7 @@ from .build import get_manifest_id from .cache import dl_build_info, gen_build_info_dir_path_prefix, touch_s3 -from .gitlab import print_section_end, print_section_start +from .gitlab import log_section from .run import runcmd from .testenv import get_bib_ref, host_container_arch, rng_seed_env @@ -227,11 +227,12 @@ def check_for_build(manifest_fname, build_request, manifest_data, build_info_dir return True +@log_section("Filtering build configurations") def filter_builds(manifests, distro=None, arch=None, skip_ostree_pull=True): """ Returns a list of build requests for the manifests that have no matching config in the test build cache. """ - print_section_start("filter-build-configs", f"⚙️ Filtering {len(manifests)} build configurations") + print(f"⚙️ Filtering {len(manifests)} build configurations") dl_root_path = os.path.join(TEST_CACHE_ROOT, "s3configs", "builds") dl_path = os.path.join(dl_root_path, gen_build_info_dir_path_prefix(distro, arch)) os.makedirs(dl_path, exist_ok=True) @@ -282,7 +283,6 @@ def filter_builds(manifests, distro=None, arch=None, skip_ostree_pull=True): print("⚠️ Errors:") print("\n".join(errors)) - print_section_end("filter-build-configs") return build_requests diff --git a/test/scripts/setup-osbuild-repo b/test/scripts/setup-osbuild-repo index 512c3acafd..859970fe16 100755 --- a/test/scripts/setup-osbuild-repo +++ b/test/scripts/setup-osbuild-repo @@ -49,6 +49,7 @@ def write_repo(commit, distro_version): repofile.write(REPO_TEMPLATE.format(commit=commit, baseurl=repo_baseurl)) +@testlib.log_section("Setting up osbuild repo") def main(): distro_version = testlib.get_host_distro() commit_id = testlib.get_osbuild_commit(distro_version) From 970c4b4bc43548439830f47a6bafa9e6e51a0e21 Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Thu, 28 May 2026 18:18:08 +0200 Subject: [PATCH 14/15] test: call setup-osbuild-repo with sudo -E The script reads environment variables to determine if it's running in Gitlab, so we need to preserve the environment when running with sudo. --- .gitlab-ci.yml | 242 +++++++++++----------- test/scripts/generate-build-config | 2 +- test/scripts/generate-gitlab-ci | 6 +- test/scripts/generate-ostree-build-config | 2 +- 4 files changed, 126 insertions(+), 126 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e763a1d55a..6f0c97f043 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,7 +62,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro centos-10 --arch aarch64 build-config.yml artifacts: @@ -77,7 +77,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro centos-10 --arch x86_64 build-config.yml artifacts: @@ -92,7 +92,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro centos-9 --arch aarch64 build-config.yml artifacts: @@ -107,7 +107,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro centos-9 --arch x86_64 build-config.yml artifacts: @@ -122,7 +122,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro eln-11 --arch aarch64 build-config.yml artifacts: @@ -137,7 +137,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro eln-11 --arch x86_64 build-config.yml artifacts: @@ -152,7 +152,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro fedora-42 --arch aarch64 build-config.yml artifacts: @@ -167,7 +167,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro fedora-42 --arch x86_64 build-config.yml artifacts: @@ -182,7 +182,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro fedora-43 --arch aarch64 build-config.yml artifacts: @@ -197,7 +197,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro fedora-43 --arch x86_64 build-config.yml artifacts: @@ -212,7 +212,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro fedora-44 --arch aarch64 build-config.yml artifacts: @@ -227,7 +227,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro fedora-44 --arch x86_64 build-config.yml artifacts: @@ -242,7 +242,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro fedora-45 --arch aarch64 build-config.yml artifacts: @@ -257,7 +257,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro fedora-45 --arch x86_64 build-config.yml artifacts: @@ -272,7 +272,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-10.0 --arch aarch64 build-config.yml artifacts: @@ -287,7 +287,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-10.0 --arch x86_64 build-config.yml artifacts: @@ -302,7 +302,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-10.1 --arch aarch64 build-config.yml artifacts: @@ -317,7 +317,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-10.1 --arch x86_64 build-config.yml artifacts: @@ -332,7 +332,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-10.2 --arch aarch64 build-config.yml artifacts: @@ -347,7 +347,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-10.2 --arch x86_64 build-config.yml artifacts: @@ -362,7 +362,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-10.3 --arch aarch64 build-config.yml artifacts: @@ -377,7 +377,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-10.3 --arch x86_64 build-config.yml artifacts: @@ -392,7 +392,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-7.9 --arch x86_64 build-config.yml artifacts: @@ -407,7 +407,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-8.10 --arch aarch64 build-config.yml artifacts: @@ -422,7 +422,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-8.10 --arch x86_64 build-config.yml artifacts: @@ -437,7 +437,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-8.4 --arch aarch64 build-config.yml artifacts: @@ -452,7 +452,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-8.4 --arch x86_64 build-config.yml artifacts: @@ -467,7 +467,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-8.6 --arch aarch64 build-config.yml artifacts: @@ -482,7 +482,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-8.6 --arch x86_64 build-config.yml artifacts: @@ -497,7 +497,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-8.8 --arch aarch64 build-config.yml artifacts: @@ -512,7 +512,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-8.8 --arch x86_64 build-config.yml artifacts: @@ -527,7 +527,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.0 --arch aarch64 build-config.yml artifacts: @@ -542,7 +542,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.0 --arch x86_64 build-config.yml artifacts: @@ -557,7 +557,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.2 --arch aarch64 build-config.yml artifacts: @@ -572,7 +572,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.2 --arch x86_64 build-config.yml artifacts: @@ -587,7 +587,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.4 --arch aarch64 build-config.yml artifacts: @@ -602,7 +602,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.4 --arch x86_64 build-config.yml artifacts: @@ -617,7 +617,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.6 --arch aarch64 build-config.yml artifacts: @@ -632,7 +632,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.6 --arch x86_64 build-config.yml artifacts: @@ -647,7 +647,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.7 --arch aarch64 build-config.yml artifacts: @@ -662,7 +662,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.7 --arch x86_64 build-config.yml artifacts: @@ -677,7 +677,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.8 --arch aarch64 build-config.yml artifacts: @@ -692,7 +692,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.8 --arch x86_64 build-config.yml artifacts: @@ -707,7 +707,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.9 --arch aarch64 build-config.yml artifacts: @@ -722,7 +722,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro rhel-9.9 --arch x86_64 build-config.yml artifacts: @@ -1230,7 +1230,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro centos-9 --arch aarch64 build-config.yml build-configs artifacts: @@ -1248,7 +1248,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro centos-9 --arch x86_64 build-config.yml build-configs artifacts: @@ -1266,7 +1266,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro fedora-42 --arch aarch64 build-config.yml build-configs artifacts: @@ -1284,7 +1284,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro fedora-42 --arch x86_64 build-config.yml build-configs artifacts: @@ -1302,7 +1302,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro fedora-43 --arch aarch64 build-config.yml build-configs artifacts: @@ -1320,7 +1320,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro fedora-43 --arch x86_64 build-config.yml build-configs artifacts: @@ -1338,7 +1338,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro fedora-44 --arch aarch64 build-config.yml build-configs artifacts: @@ -1356,7 +1356,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro fedora-44 --arch x86_64 build-config.yml build-configs artifacts: @@ -1374,7 +1374,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro fedora-45 --arch aarch64 build-config.yml build-configs artifacts: @@ -1392,7 +1392,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro fedora-45 --arch x86_64 build-config.yml build-configs artifacts: @@ -1410,7 +1410,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-8.10 --arch aarch64 build-config.yml build-configs artifacts: @@ -1428,7 +1428,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-8.10 --arch x86_64 build-config.yml build-configs artifacts: @@ -1446,7 +1446,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-8.4 --arch aarch64 build-config.yml build-configs artifacts: @@ -1464,7 +1464,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-8.4 --arch x86_64 build-config.yml build-configs artifacts: @@ -1482,7 +1482,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-8.6 --arch aarch64 build-config.yml build-configs artifacts: @@ -1500,7 +1500,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-8.6 --arch x86_64 build-config.yml build-configs artifacts: @@ -1518,7 +1518,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-8.8 --arch aarch64 build-config.yml build-configs artifacts: @@ -1536,7 +1536,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-8.8 --arch x86_64 build-config.yml build-configs artifacts: @@ -1554,7 +1554,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.0 --arch aarch64 build-config.yml build-configs artifacts: @@ -1572,7 +1572,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.0 --arch x86_64 build-config.yml build-configs artifacts: @@ -1590,7 +1590,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.2 --arch aarch64 build-config.yml build-configs artifacts: @@ -1608,7 +1608,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.2 --arch x86_64 build-config.yml build-configs artifacts: @@ -1626,7 +1626,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.4 --arch aarch64 build-config.yml build-configs artifacts: @@ -1644,7 +1644,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.4 --arch x86_64 build-config.yml build-configs artifacts: @@ -1662,7 +1662,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.6 --arch aarch64 build-config.yml build-configs artifacts: @@ -1680,7 +1680,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.6 --arch x86_64 build-config.yml build-configs artifacts: @@ -1698,7 +1698,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.7 --arch aarch64 build-config.yml build-configs artifacts: @@ -1716,7 +1716,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.7 --arch x86_64 build-config.yml build-configs artifacts: @@ -1734,7 +1734,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.8 --arch aarch64 build-config.yml build-configs artifacts: @@ -1752,7 +1752,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.8 --arch x86_64 build-config.yml build-configs artifacts: @@ -1770,7 +1770,7 @@ fail: RUNNER: aws/fedora-42-aarch64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.9 --arch aarch64 build-config.yml build-configs artifacts: @@ -1788,7 +1788,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro rhel-9.9 --arch x86_64 build-config.yml build-configs artifacts: @@ -2156,7 +2156,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros centos-10 --workers 10 --metadata=false --output ./manifests @@ -2180,7 +2180,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros centos-10 --workers 10 --metadata=false --output ./manifests @@ -2204,7 +2204,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros centos-9 --workers 10 --metadata=false --output ./manifests @@ -2228,7 +2228,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros centos-9 --workers 10 --metadata=false --output ./manifests @@ -2252,7 +2252,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros eln-11 --workers 10 --metadata=false --output ./manifests @@ -2276,7 +2276,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros eln-11 --workers 10 --metadata=false --output ./manifests @@ -2300,7 +2300,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros fedora-42 --workers 10 --metadata=false --output ./manifests @@ -2324,7 +2324,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros fedora-42 --workers 10 --metadata=false --output ./manifests @@ -2348,7 +2348,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros fedora-43 --workers 10 --metadata=false --output ./manifests @@ -2372,7 +2372,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros fedora-43 --workers 10 --metadata=false --output ./manifests @@ -2396,7 +2396,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros fedora-44 --workers 10 --metadata=false --output ./manifests @@ -2420,7 +2420,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros fedora-44 --workers 10 --metadata=false --output ./manifests @@ -2444,7 +2444,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros fedora-45 --workers 10 --metadata=false --output ./manifests @@ -2468,7 +2468,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros fedora-45 --workers 10 --metadata=false --output ./manifests @@ -2492,7 +2492,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-10.0 --workers 10 --metadata=false --output ./manifests @@ -2516,7 +2516,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-10.0 --workers 10 --metadata=false --output ./manifests @@ -2540,7 +2540,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-10.1 --workers 10 --metadata=false --output ./manifests @@ -2564,7 +2564,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-10.1 --workers 10 --metadata=false --output ./manifests @@ -2588,7 +2588,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-10.2 --workers 10 --metadata=false --output ./manifests @@ -2612,7 +2612,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-10.2 --workers 10 --metadata=false --output ./manifests @@ -2636,7 +2636,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-10.3 --workers 10 --metadata=false --output ./manifests @@ -2660,7 +2660,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-10.3 --workers 10 --metadata=false --output ./manifests @@ -2684,7 +2684,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.10 --workers 10 --metadata=false --output ./manifests @@ -2708,7 +2708,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-8.10 --workers 10 --metadata=false --output ./manifests @@ -2732,7 +2732,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.4 --workers 10 --metadata=false --output ./manifests @@ -2756,7 +2756,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-8.4 --workers 10 --metadata=false --output ./manifests @@ -2780,7 +2780,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.6 --workers 10 --metadata=false --output ./manifests @@ -2804,7 +2804,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-8.6 --workers 10 --metadata=false --output ./manifests @@ -2828,7 +2828,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-8.8 --workers 10 --metadata=false --output ./manifests @@ -2852,7 +2852,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-8.8 --workers 10 --metadata=false --output ./manifests @@ -2876,7 +2876,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.0 --workers 10 --metadata=false --output ./manifests @@ -2900,7 +2900,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-9.0 --workers 10 --metadata=false --output ./manifests @@ -2924,7 +2924,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.2 --workers 10 --metadata=false --output ./manifests @@ -2948,7 +2948,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-9.2 --workers 10 --metadata=false --output ./manifests @@ -2972,7 +2972,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.4 --workers 10 --metadata=false --output ./manifests @@ -2996,7 +2996,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-9.4 --workers 10 --metadata=false --output ./manifests @@ -3020,7 +3020,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.6 --workers 10 --metadata=false --output ./manifests @@ -3044,7 +3044,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-9.6 --workers 10 --metadata=false --output ./manifests @@ -3068,7 +3068,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.7 --workers 10 --metadata=false --output ./manifests @@ -3092,7 +3092,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-9.7 --workers 10 --metadata=false --output ./manifests @@ -3116,7 +3116,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.8 --workers 10 --metadata=false --output ./manifests @@ -3140,7 +3140,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-9.8 --workers 10 --metadata=false --output ./manifests @@ -3164,7 +3164,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches ppc64le --distros rhel-9.9 --workers 10 --metadata=false --output ./manifests @@ -3188,7 +3188,7 @@ fail: RUNNER: aws/fedora-42-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches s390x --distros rhel-9.9 --workers 10 --metadata=false --output ./manifests diff --git a/test/scripts/generate-build-config b/test/scripts/generate-build-config index 6086b99ac4..71a60bd5d3 100755 --- a/test/scripts/generate-build-config +++ b/test/scripts/generate-build-config @@ -11,7 +11,7 @@ JOB_TEMPLATE = """ build/{distro}/{arch}/{image_type}/{config_name}: stage: test script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/build-image "{distro}" "{image_type}" "{config}" - ./test/scripts/boot-image "{image_path}" "{config}" diff --git a/test/scripts/generate-gitlab-ci b/test/scripts/generate-gitlab-ci index 5786d7f0c2..070852785c 100755 --- a/test/scripts/generate-gitlab-ci +++ b/test/scripts/generate-gitlab-ci @@ -61,7 +61,7 @@ GEN_TEMPLATE = """ RUNNER: {runner}-{arch} INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-build-config --distro {distro} --arch {arch} build-config.yml artifacts: @@ -89,7 +89,7 @@ OSTREE_GEN_TEMPLATE = """ RUNNER: {runner}-{arch} INTERNAL_NETWORK: "true" script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - ./test/scripts/generate-ostree-build-config --distro {distro} --arch {arch} build-config.yml build-configs artifacts: @@ -121,7 +121,7 @@ MANIFEST_GEN_TEMPLATE = """ RUNNER: {runner}-x86_64 INTERNAL_NETWORK: "true" script: | - sudo ./test/scripts/setup-osbuild-repo + sudo -E ./test/scripts/setup-osbuild-repo sudo ./test/scripts/install-dependencies echo "GOPROXY=$GOPROXY" go run ./cmd/gen-manifests --arches {arch} --distros {distro} --workers 10 --metadata=false --output ./manifests diff --git a/test/scripts/generate-ostree-build-config b/test/scripts/generate-ostree-build-config index 3ecddc8304..e8ec3c512c 100755 --- a/test/scripts/generate-ostree-build-config +++ b/test/scripts/generate-ostree-build-config @@ -11,7 +11,7 @@ JOB_TEMPLATE = """ build/{distro}/{arch}/{image_type}/{config_name}: stage: test script: - - sudo ./test/scripts/setup-osbuild-repo + - sudo -E ./test/scripts/setup-osbuild-repo - sudo ./test/scripts/install-dependencies - {dl_container} - {start_container} From 6b3234f6e1fe3da45e69341769f5aa5e3c07ccfa Mon Sep 17 00:00:00 2001 From: Achilleas Koutsou Date: Fri, 29 May 2026 12:02:48 +0200 Subject: [PATCH 15/15] test/imgtestlib: skip network-installer boot test on 10.3 The beta repos are showing the same behaviour that was discovered before with 10.1 and 10.2. https://github.com/osbuild/images/pull/2044#issue-3667039388 --- test/scripts/imgtestlib/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/scripts/imgtestlib/core.py b/test/scripts/imgtestlib/core.py index 380465316a..b4d64cd5a2 100644 --- a/test/scripts/imgtestlib/core.py +++ b/test/scripts/imgtestlib/core.py @@ -402,7 +402,7 @@ def can_boot_test(manifest_fname, manifest_data, image_type, arch, distro, bluep return False if image_type in ["network-installer", "everything-network-installer", "server-network-installer"]: - if distro in ["rhel-10.1", "rhel-10.2"]: + if distro in ["rhel-10.1", "rhel-10.2", "rhel-10.3"]: print(" not bootable: rhel network-installer tests have incomplete repos in nightly snapshot" "and won't install") return False