spread.yaml

# spread-variant=plus
project: snapd

environment:
    GOHOME: /home/gopath
    GOPATH: $GOHOME
    # On some distros the default GOPROXY setting is 'direct' (eg. Fedora).
    # Some of the external go packages may get removed or could be temporary
    # offline (as happened with maze.io/x/crypt), and then the sources are
    # only available through the proxy cache. Play it safe and enable the
    # proxy to allow for least CI interruptions.
    GOPROXY: https://proxy.golang.org,direct
    REUSE_PROJECT: '$(HOST: echo "$REUSE_PROJECT")'
    PROJECT_PATH: $GOHOME/src/github.com/snapcore/snapd
    PATH: $PROJECT_PATH/tests/bin:$GOHOME/bin:/snap/bin:$PATH:/var/lib/snapd/snap/bin:/usr/lib/python
    TESTSLIB: $PROJECT_PATH/tests/lib
    TESTSTOOLS: $PROJECT_PATH/tests/lib/tools
    TESTSTMP: /var/tmp/snapd-tools
    RUNTIME_STATE_PATH: $TESTSTMP/runtime-state
    # turn debug off so that we don't get errant debug messages while running
    # tests, and in some cases like on UC20 we have the kernel command line
    # parameter, snapd.debug=1 turned on to enable early boot debugging before
    # we have a shell, but then once we get a shell and run spread tests, we
    # want debug messages to be off for commands we run as part of tests, unless
    # tests explicitly turn the messages on
    SNAPD_DEBUG: 0
    SNAPPY_TESTING: 1
    # we run the entire suite with re-exec on (the default) and modify
    # the core snap so that it contains our new code.  So we run new
    # snapd from the deb that re-execs into new snapd in core.  To
    # test purely from the deb, set "export SPREAD_SNAP_REEXEC=0"
    SNAP_REEXEC: '$(HOST: echo "${SPREAD_SNAP_REEXEC:-}")'
    MODIFY_CORE_SNAP_FOR_REEXEC: '$(HOST: echo "${SPREAD_MODIFY_CORE_SNAP_FOR_REEXEC:-1}")'
    SPREAD_STORE_USER: '$(HOST: echo "$SPREAD_STORE_USER")'
    SPREAD_STORE_PASSWORD: '$(HOST: echo "$SPREAD_STORE_PASSWORD")'
    SPREAD_STORE_EXPIRED_MACAROON: '$(HOST: echo "$SPREAD_STORE_EXPIRED_MACAROON")'
    SPREAD_STORE_EXPIRED_DISCHARGE: '$(HOST: echo "$SPREAD_STORE_EXPIRED_DISCHARGE")'
    SPREAD_DEBUG_EACH: '$(HOST: echo "${SPREAD_DEBUG_EACH:-1}")'
    LANG: "C.UTF-8"
    LANGUAGE: "en"
    # important to ensure adhoc and linode/qemu behave the same
    SUDO_USER: ""
    SUDO_UID: ""
    TRUST_TEST_KEYS: '$(HOST: echo "${SPREAD_TRUST_TEST_KEYS:-true}")'
    # a global setting for LXD channel to use in the tests
    LXD_SNAP_CHANNEL: "latest/candidate"
    OLD_UBUNTU_IMAGE_SNAP_CHANNEL: "2/stable"
    UBUNTU_IMAGE_SNAP_CHANNEL: "latest/beta"
    SNAPCRAFT_SNAP_CHANNEL: "latest/stable"
    # controls whether ubuntu-image is built using the current snapd tree as a
    # dependency or the one listed in its go.mod
    UBUNTU_IMAGE_ALLOW_API_BREAK: '$(HOST: echo "${SPREAD_UBUNTU_IMAGE_ALLOW_API_BREAK:-true}")'
    CORE_CHANNEL: '$(HOST: echo "${SPREAD_CORE_CHANNEL:-edge}")'
    BASE_CHANNEL: '$(HOST: echo "${SPREAD_BASE_CHANNEL:-edge}")'
    # Channel for kernel cannot be edge as that one is not signed with Canonical keys
    KERNEL_CHANNEL: '$(HOST: echo "${SPREAD_KERNEL_CHANNEL:-beta}")'
    GADGET_CHANNEL: '$(HOST: echo "${SPREAD_GADGET_CHANNEL:-beta}")'
    SNAPD_CHANNEL: '$(HOST: echo "${SPREAD_SNAPD_CHANNEL:-edge}")'
    REMOTE_STORE: '$(HOST: echo "${SPREAD_REMOTE_STORE:-production}")'
    SNAPPY_USE_STAGING_STORE: '$(HOST: if [ "$SPREAD_REMOTE_STORE" = staging ]; then echo 1; else echo 0; fi)'
    DELTA_REF: 2.62
    DELTA_PREFIX: snapd-$DELTA_REF/
    REPACK_KEEP_VENDOR: '$(HOST: echo "${REPACK_KEEP_VENDOR:-n}")'
    HTTP_PROXY: '$(HOST: echo "$SPREAD_HTTP_PROXY")'
    http_proxy: '$(HOST: echo "$SPREAD_HTTP_PROXY")'
    HTTPS_PROXY: '$(HOST: echo "$SPREAD_HTTPS_PROXY")'
    https_proxy: '$(HOST: echo "$SPREAD_HTTPS_PROXY")'
    NO_PROXY: '$(HOST: echo "${SPREAD_NO_PROXY:-127.0.0.1}")'
    no_proxy: '$(HOST: echo "${SPREAD_NO_PROXY:-127.0.0.1}")'
    # use NEW_CORE_CHANNEL to control the core channel used in refresh tests
    NEW_CORE_CHANNEL: '$(HOST: echo "$SPREAD_NEW_CORE_CHANNEL")'
    # use NEW_SNAPD_CHANNEL to control the snapd channel used in refresh tests
    NEW_SNAPD_CHANNEL: '$(HOST: echo "${SPREAD_NEW_SNAPD_CHANNEL:-edge}")'
    SRU_VALIDATION: '$(HOST: echo "${SPREAD_SRU_VALIDATION:-0}")'
    # use the ppa_validation_name to install snapd from a public ppa
    PPA_VALIDATION_NAME: '$(HOST: echo "${SPREAD_PPA_VALIDATION_NAME:-}")'
    # use the ppa_source_line and ppa_gpg_key to install snapd from a private ppa
    PPA_SOURCE_LINE: '$(HOST: echo "${SPREAD_PPA_SOURCE_LINE:-}")'
    PPA_GPG_KEY: '$(HOST: echo "${SPREAD_PPA_GPG_KEY:-}")'
    # List the snaps which are cached
    PRE_CACHE_SNAPS: test-snapd-tools test-snapd-sh
    # always skip removing the rsync snap
    SKIP_REMOVE_SNAPS: '$(HOST: echo "${SPREAD_SKIP_REMOVE_SNAPS:-}") test-snapd-rsync test-snapd-rsync-core18 test-snapd-rsync-core20 test-snapd-rsync-core22 test-snapd-rsync-core24 test-snapd-rsync-core26'
    # Use the installed snapd and reset the systems without removing snapd
    REUSE_SNAPD: '$(HOST: echo "${SPREAD_REUSE_SNAPD:-0}")'
    EXPERIMENTAL_FEATURES: '$(HOST: echo "${SPREAD_EXPERIMENTAL_FEATURES:-}")'
    # set to 1 when the snapd memory limit has to be removed
    SNAPD_NO_MEMORY_LIMIT: '$(HOST: echo "${SPREAD_SNAPD_NO_MEMORY_LIMIT:-}")'

    # Use the snapd package from the repository when possible
    SNAPD_DEB_FROM_REPO: '$(HOST: echo "${SPREAD_SNAPD_DEB_FROM_REPO:-true}")'
    # Directory where the built snapd snaps and other assets are stored
    SNAPD_WORK_DIR: '$(HOST: echo "${SPREAD_SNAPD_WORK_DIR:-/var/tmp/work-dir}")'
    # Threshold used to determine which stake locks have to be collected during runtime
    SNAPD_STATE_LOCK_TRACE_THRESHOLD_MS: '$(HOST: echo "${SPREAD_SNAPD_STATE_LOCK_TRACE_THRESHOLD_MS:-0}")'
    # Allow mismatch in snapd/kernel versions
    SNAPD_ALLOW_SNAPD_KERNEL_MISMATCH: '$(HOST: echo "${SNAPD_ALLOW_SNAPD_KERNEL_MISMATCH:-true}")'
    # The pattern used by `apt-cache search` to determine the ubuntu kernel to be used in tests
    UPDATE_UBUNTU_KERNEL_PATTERN: '$(HOST: echo "${SPREAD_UPDATE_UBUNTU_KERNEL_PATTERN:-}")'

    # Directory where the nested images and test assets are stored
    NESTED_WORK_DIR: '$(HOST: echo "${NESTED_WORK_DIR:-/var/tmp/work-dir}")'
    # Channel used to create the nested vm
    NESTED_CORE_CHANNEL: '$(HOST: echo "${NESTED_CORE_CHANNEL:-beta}")'
    # Kernel channel used to create the nested vm
    NESTED_KERNEL_CHANNEL: '$(HOST: echo "${NESTED_KERNEL_CHANNEL:-beta}")'
    # Gadget channel used to create the nested vm
    NESTED_GADGET_CHANNEL: '$(HOST: echo "${NESTED_GADGET_CHANNEL:-edge}")'
    # Use cloud init to make initial system configuration instead of user assertion
    NESTED_CORE_REFRESH_CHANNEL: '$(HOST: echo "${NESTED_CORE_REFRESH_CHANNEL:-edge}")'
    # Use cloud init to make initial system configuration instead of user assertion
    NESTED_USE_CLOUD_INIT: '$(HOST: echo "${NESTED_USE_CLOUD_INIT:-true}")'
    # Build and use snapd from current branch
    NESTED_BUILD_SNAPD_FROM_CURRENT: '$(HOST: echo "${NESTED_BUILD_SNAPD_FROM_CURRENT:-true}")'
    # Download and use an custom image from this url
    NESTED_CUSTOM_IMAGE_URL: '$(HOST: echo "${NESTED_CUSTOM_IMAGE_URL:-}")'
    # Configure nested images to be reused on the following tests
    NESTED_CONFIGURE_IMAGES: '$(HOST: echo "${NESTED_CONFIGURE_IMAGES:-false}")'
    # Indicates if the snap has to be repacked in case NESTED_BUILD_SNAPD_FROM_CURRENT is true
    NESTED_REPACK_KERNEL_SNAP: '$(HOST: echo "${NESTED_REPACK_KERNEL_SNAP:-true}")'
    NESTED_REPACK_GADGET_SNAP: '$(HOST: echo "${NESTED_REPACK_GADGET_SNAP:-true}")'
    NESTED_REPACK_BASE_SNAP: '$(HOST: echo "${NESTED_REPACK_BASE_SNAP:-true}")'
    NESTED_KERNEL_MODULES_COMP: '$(HOST: echo "${NESTED_KERNEL_MODULES_COMP:-}")'

    # Whether we should use snapd snap ./built-snap/ directory
    USE_PREBUILT_SNAPD_SNAP: '$(HOST: echo "${SPREAD_USE_PREBUILT_SNAPD_SNAP:-false}")'
    SNAPD_USE_PROXY: '$(HOST: echo "${SPREAD_SNAPD_USE_PROXY:-}")'

    # Whether we should use pre-built packages in the ./built-pkgs/ directory
    USE_PREBUILT_PACKAGES: '$(HOST: echo "${SPREAD_USE_PREBUILT_PACKAGES:-false}")'

    # When SPREAD_TAG_FEATURES is populated with a non-empty value,
    # each spread task will extract relevant information form its
    # journal and copy it and the state.json to the artifacts folder.
    # NOTE: this should only be used coupled with the -artifact spread
    # tag otherwise the data will not be downloaded from the VMs.
    TAG_FEATURES: '$(HOST: echo "${SPREAD_TAG_FEATURES:-}")'
    # Set the following to SPREAD_TAG_FEATURES by default
    # since the following are required to log snap commands
    # to the journal
    SNAPD_LOG_SNAP_TO_JOURNAL: '$(HOST: echo "${SPREAD_SNAPD_LOG_SNAP_TO_JOURNAL:-$SPREAD_TAG_FEATURES}")'
    SNAPD_TRACE: '$(HOST: echo "${SPREAD_SNAPD_TRACE:-$SPREAD_TAG_FEATURES}")'

    # Whether snapd should skip early refreshes.
    SNAPD_SKIP_EARLY_REFRESH: '$(HOST: echo "${SPREAD_SNAPD_SKIP_EARLY_REFRESH:-}")'

backends:
    google:
        key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
        location: snapd-spread/europe-west2-b
        halt-timeout: 2h
        systems:
            - ubuntu-14.04-64:
                  workers: 6
            - ubuntu-16.04-64:
                  workers: 8
                  storage: 15G
            - ubuntu-18.04-64:
                  storage: 15G
                  workers: 8
            - ubuntu-20.04-64:
                  image: ubuntu-2004-64
                  storage: 16G
                  workers: 8
            - ubuntu-secboot-20.04-64:
                  image: ubuntu-2004-64
                  storage: 16G
                  workers: 1
                  secure-boot: true
            - ubuntu-22.04-64:
                  image: ubuntu-2204-64
                  storage: 15G
                  workers: 8
            - ubuntu-24.04-64:
                  image: ubuntu-2404-64
                  storage: 15G
                  workers: 8
            - ubuntu-25.10-64:
                  image: ubuntu-os-cloud-devel/ubuntu-2510-amd64
                  storage: 15G
                  workers: 8

    google-core:
        type: google
        key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
        location: snapd-spread/europe-west1-b
        halt-timeout: 2h
        systems:
            - ubuntu-core-16-64:
                  image: ubuntu-16.04-64
                  workers: 6
                  storage: 20G
            - ubuntu-core-18-64:
                  image: ubuntu-18.04-64
                  workers: 6
                  storage: 20G
            - ubuntu-core-20-64:
                  image: ubuntu-2004-64
                  workers: 8
                  storage: 20G
            - ubuntu-core-22-64:
                  image: ubuntu-2204-64
                  workers: 8
                  storage: 20G
            - ubuntu-core-24-64:
                  image: ubuntu-2404-64
                  workers: 8
                  storage: 20G
            - ubuntu-core-26-64:
                  image: ubuntu-2604-64
                  workers: 8
                  storage: 20G

    google-pro:
        type: google
        # XXX the role assigned to the key must allow for attaching service
        # accounts
        key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
        location: snapd-spread/europe-west1-b
        halt-timeout: 2h
        systems:
            - ubuntu-pro-20.04-64:
                  image: ubuntu-os-pro-cloud/ubuntu-pro-2004-lts
                  workers: 1
                  attach-service-account: true

            - ubuntu-pro-22.04-64:
                  image: ubuntu-os-pro-cloud/ubuntu-pro-2204-lts
                  workers: 1
                  attach-service-account: true

            - ubuntu-fips-20.04-64:
                  image: ubuntu-os-pro-cloud/ubuntu-pro-2004-lts
                  workers: 1
                  attach-service-account: true

            - ubuntu-fips-22.04-64:
                  image: ubuntu-os-pro-cloud/ubuntu-pro-2204-lts
                  workers: 1
                  attach-service-account: true

            - ubuntu-fips-24.04-64:
                  image: ubuntu-os-pro-cloud/ubuntu-pro-2404-lts-amd64
                  workers: 1
                  attach-service-account: true

    google-distro:
        type: google
        key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
        location: snapd-spread/europe-west3-b
        halt-timeout: 2h
        systems:
            - debian-12-64:
                  storage: 20G
                  workers: 6
            - debian-sid-64:
                  storage: 20G
                  workers: 6

            - amazon-linux-2-64:
                  workers: 6
                  storage: preserve-size
            - amazon-linux-2023-64:
                  workers: 6
                  storage: preserve-size

            - arch-linux-64:
                  workers: 6
                  storage: 20G

            - centos-9-64:
                  workers: 6
                  storage: preserve-size
                  image: centos-9-64

            - opensuse-16.0-64:
                  workers: 6
            - opensuse-tumbleweed-64:
                  workers: 6
                  storage: 12G
            - opensuse-tumbleweed-selinux-64:
                  image: snapd-spread/opensuse-tumbleweed-64-selinux-enabled
                  workers: 6
                  storage: 12G

    google-arm:
        type: google
        key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
        location: snapd-spread/us-central1-a
        plan: t2a-standard-1
        halt-timeout: 2h
        kill-timeout: 60m
        systems:
            - ubuntu-20.04-arm-64:
                  image: ubuntu-os-cloud/ubuntu-2004-lts-arm64
                  workers: 8
                  storage: 15G
            - ubuntu-22.04-arm-64:
                  image: ubuntu-os-cloud/ubuntu-2204-lts-arm64
                  workers: 8
                  storage: 15G
            - ubuntu-24.04-arm-64:
                  image: ubuntu-os-cloud/ubuntu-2404-lts-arm64
                  workers: 8
                  storage: 15G
            - ubuntu-core-22-arm-64:
                  image: ubuntu-22.04-arm-64
                  workers: 6
                  storage: 30G
            - ubuntu-core-24-arm-64:
                  image: ubuntu-24.04-arm-64
                  workers: 6
                  storage: 30G
    # TODO add garden-sru backend
    google-nested:
        type: google
        key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
        location: snapd-spread/us-central1-a
        plan: n2-standard-2
        halt-timeout: 2h
        environment:
            NESTED_ARCHITECTURE: amd64
        systems: &google-nested-systems
            - ubuntu-16.04-64:
                  image: ubuntu-1604-64-virt-enabled
                  storage: 20G
                  workers: 4
            - ubuntu-18.04-64:
                  image: ubuntu-1804-64-virt-enabled
                  storage: 20G
                  workers: 4
            - ubuntu-20.04-64:
                  image: ubuntu-2004-64-virt-enabled
                  storage: 20G
                  workers: 12
            - ubuntu-22.04-64:
                  image: ubuntu-2204-64-virt-enabled
                  storage: 25G
                  workers: 14
            - ubuntu-24.04-64:
                  image: ubuntu-2404-64-virt-enabled
                  storage: 30G
                  workers: 14
            - ubuntu-26.04-64:
                  image: ubuntu-2604-64-virt-enabled
                  storage: 30G
                  workers: 14

    google-nested-arm:
        type: google
        key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
        location: snapd-spread/us-central1-a
        plan: c4a-highcpu-4
        halt-timeout: 2h
        environment:
            NESTED_ARCHITECTURE: arm64
            NESTED_ENABLE_KVM: false
            NESTED_ENABLE_TPM: '$(HOST: echo "${NESTED_ENABLE_TPM:-false}")'
            NESTED_ENABLE_SECURE_BOOT: '$(HOST: echo "${NESTED_ENABLE_SECURE_BOOT:-false}")'
        systems:
            - ubuntu-22.04-arm-64:
                  image: ubuntu-2204-arm-64-virt-enabled
                  storage: 25G
                  workers: 8
            - ubuntu-24.04-arm-64:
                  image: ubuntu-2404-arm-64-virt-enabled
                  storage: 25G
                  workers: 8

    openstack:
        key: '$(HOST: echo "$OS_CREDENTIALS_AMD64_PS7")'
        plan: shared.xsmall
        halt-timeout: 2h
        wait-timeout: 10m
        groups: [default]
        proxy: ingress-haproxy.ps7.canonical.com
        cidr-port-rel: [10.151.96.0/21:5000]
        volume-auto-delete: true
        environment: &openstack-env
            SNAPD_USE_PROXY: 'true'
            HTTP_PROXY: 'http://egress.ps7.internal:3128'
            HTTPS_PROXY: 'http://egress.ps7.internal:3128'
            http_proxy: 'http://egress.ps7.internal:3128'
            https_proxy: 'http://egress.ps7.internal:3128'
            no_proxy: '127.0.0.1,127.0.0.53,localhost'
            NO_PROXY: '127.0.0.1,127.0.0.53,localhost'
            NTP_SERVER: 'ntp.ps7.internal'
        systems: &openstack-systems
            - ubuntu-16.04-64:
                image: snapd-spread/ubuntu-16.04-64
                workers: 8
            - ubuntu-18.04-64:
                image: snapd-spread/ubuntu-18.04-64
                workers: 8
            - ubuntu-20.04-64:
                image: snapd-spread/ubuntu-20.04-64
                workers: 8
            - ubuntu-22.04-64:
                image: snapd-spread/ubuntu-22.04-64
                workers: 8
            - ubuntu-fips-22.04-64:
                image: snapd-spread/ubuntu-22.04-64-pro-fips
            - ubuntu-22.04-64:
                image: snapd-spread/ubuntu-22.04-64
                workers: 8
            - ubuntu-24.04-64:
                image: snapd-spread/ubuntu-24.04-64
                workers: 8
            - ubuntu-fips-24.04-64:
                image: snapd-spread/ubuntu-24.04-64-pro-fips
            - ubuntu-secboot-24.04-64:
                image: snapd-spread/ubuntu-24.04-64-secboot
            - ubuntu-25.10-64:
                image: snapd-spread/ubuntu-25.10-64
                workers: 8
            - ubuntu-26.04-64:
                image: snapd-spread/ubuntu-26.04-64
                workers: 8

            - ubuntu-core-18-64:
                image: snapd-spread/ubuntu-18.04-64
                workers: 8
            - ubuntu-core-20-64:
                image: snapd-spread/ubuntu-20.04-64-uefi
                workers: 8
            - ubuntu-core-22-64:
                image: snapd-spread/ubuntu-22.04-64-uefi
                workers: 8
            - ubuntu-core-24-64:
                image: snapd-spread/ubuntu-24.04-64-uefi
                workers: 8
            - ubuntu-core-26-64:
                image: snapd-spread/ubuntu-26.04-64-uefi
                workers: 8

            - debian-12-64:
                image: snapd-spread/debian-12-64
                workers: 6
            - debian-sid-64:
                image: snapd-spread/debian-sid-64
                workers: 6

            - amazon-linux-2-64:
                image: snapd-spread/amazon-linux-2-64
                workers: 6
                storage: 50G
            - amazon-linux-2023-64:
                image: snapd-spread/amazon-linux-2023-64
                workers: 6
                storage: 50G

            - arch-linux-64:
                image: snapd-spread/arch-linux-64
                workers: 6

            - fedora-44-64:
                  image: snapd-spread/fedora-44-64
                  workers: 12

            - opensuse-16.0-64:
                image: snapd-spread/opensuse-16.0-64
                workers: 6
            - opensuse-tumbleweed-64:
                image: snapd-spread/opensuse-tumbleweed-64
                workers: 6
            - opensuse-tumbleweed-selinux-64:
                  image: snapd-spread/opensuse-tumbleweed-64-selinux-enabled
                  workers: 6

            - centos-9-64:
                image: snapd-spread/centos-9-64
                workers: 6

    openstack-dev:
        type: openstack
        key: '$(HOST: echo "$OS_CREDENTIALS_STG_AMD64_PS7")'
        plan: shared.xsmall
        halt-timeout: 2h
        wait-timeout: 10m
        groups: [default]
        proxy: ingress-haproxy.ps7.canonical.com
        cidr-port-rel: [10.151.54.0/24:4000]
        volume-auto-delete: true
        environment: *openstack-env
        systems: *openstack-systems

    openstack-ext:
        type: openstack
        key: '$(HOST: echo "$OS_CREDENTIALS_AMD64_PS7")'
        plan: shared.medium
        storage: 30G
        halt-timeout: 2h
        wait-timeout: 10m
        groups: [default]
        proxy: ingress-haproxy.ps7.canonical.com
        cidr-port-rel: [10.151.96.0/21:5000]
        volume-auto-delete: true
        environment: *openstack-env
        systems: &openstack-nested-systems
            - ubuntu-16.04-64:
                  image: snapd-spread/ubuntu-16.04-64
                  storage: 20G
                  workers: 4
            - ubuntu-18.04-64:
                  image: snapd-spread/ubuntu-18.04-64
                  storage: 20G
                  workers: 4
            - ubuntu-20.04-64:
                  image: snapd-spread/ubuntu-20.04-64-hwe
                  storage: 20G
                  workers: 12
            - ubuntu-22.04-64:
                  image: snapd-spread/ubuntu-22.04-64
                  storage: 25G
                  workers: 12
            - ubuntu-24.04-64:
                  image: snapd-spread/ubuntu-24.04-64
                  storage: 30G
                  workers: 12
            - ubuntu-26.04-64:
                  image: snapd-spread/ubuntu-26.04-64
                  storage: 30G
                  workers: 12

    openstack-ext-dev:
        type: openstack
        key: '$(HOST: echo "$OS_CREDENTIALS_STG_AMD64_PS7")'
        plan: shared.medium
        storage: 30G
        halt-timeout: 2h
        wait-timeout: 10m
        groups: [default]
        proxy: ingress-haproxy.ps7.canonical.com
        cidr-port-rel: [10.151.54.0/24:4000]
        volume-auto-delete: true
        environment: *openstack-env
        systems: *openstack-nested-systems

    openstack-validation:
        type: openstack
        key: '$(HOST: echo "$OS_CREDENTIALS_AMD64_PS7")'
        plan: shared.medium
        storage: 30G
        halt-timeout: 6h
        wait-timeout: 10m
        groups: [default]
        proxy: ingress-haproxy.ps7.canonical.com
        cidr-port-rel: [10.151.96.0/21:5000]
        volume-auto-delete: true
        environment: *openstack-env
        systems: *openstack-nested-systems

    openstack-arm:
        type: openstack
        key: '$(HOST: echo "$OS_CREDENTIALS_ARM64_PS7")'
        plan: shared.xsmall.arm64
        halt-timeout: 2h
        wait-timeout: 10m
        groups: [default]
        proxy: ingress-haproxy.ps7.canonical.com
        cidr-port-rel: [10.151.89.0/24:8000]
        volume-auto-delete: true
        environment: *openstack-env
        systems: &openstack-arm-systems
            - ubuntu-20.04-arm-64:
                image: snapd-spread/ubuntu-20.04-arm-64
                workers: 8
            - ubuntu-22.04-arm-64:
                image: snapd-spread/ubuntu-22.04-arm-64
                workers: 8
            - ubuntu-24.04-arm-64:
                image: snapd-spread/ubuntu-24.04-arm-64
                workers: 8
            - ubuntu-26.04-arm-64:
                image: snapd-spread/ubuntu-26.04-arm-64
                workers: 8

            - ubuntu-core-22-arm-64:
                image: snapd-spread/ubuntu-22.04-arm-64-uefi
                workers: 8
            - ubuntu-core-24-arm-64:
                image: snapd-spread/ubuntu-24.04-arm-64-uefi
                workers: 8
            - ubuntu-core-26-arm-64:
                image: snapd-spread/ubuntu-26.04-arm-64-uefi
                workers: 8

    openstack-arm-ext:
        type: openstack
        key: '$(HOST: echo "$OS_CREDENTIALS_ARM64_PS7")'
        plan: shared.large.arm64
        halt-timeout: 3h
        wait-timeout: 10m
        groups: [default]
        proxy: ingress-haproxy.ps7.canonical.com
        cidr-port-rel: [10.151.89.0/24:8000]
        volume-auto-delete: true
        environment: *openstack-env
        systems: *openstack-arm-systems

    openstack-arm-dev:
        type: openstack
        key: '$(HOST: echo "$OS_CREDENTIALS_STG_ARM64_PS7")'
        plan: shared.xsmall.arm64
        halt-timeout: 2h
        wait-timeout: 10m
        groups: [default]
        proxy: ingress-haproxy.ps7.canonical.com
        cidr-port-rel: [10.151.53.0/24:3000]
        volume-auto-delete: true
        environment: *openstack-env
        systems: *openstack-arm-systems

    openstack-sru:
        type: openstack
        key: '$(HOST: echo "$OS_CREDENTIALS_AMD64_PS7")'
        plan: shared.xsmall
        halt-timeout: 2h
        wait-timeout: 10m
        groups: [default]
        proxy: ingress-haproxy.ps7.canonical.com
        cidr-port-rel: [10.151.96.0/21:5000]
        volume-auto-delete: true
        environment:
            SNAPD_USE_PROXY: 'true'
            HTTP_PROXY: 'http://egress.ps7.internal:3128'
            HTTPS_PROXY: 'http://egress.ps7.internal:3128'
            http_proxy: 'http://egress.ps7.internal:3128'
            https_proxy: 'http://egress.ps7.internal:3128'
            no_proxy: '127.0.0.1,127.0.0.53,localhost'
            NO_PROXY: '127.0.0.1,127.0.0.53,localhost'
            NTP_SERVER: 'ntp.ps7.internal'
            SRU_VALIDATION: 1
            SNAP_REEXEC: 0
            MODIFY_CORE_SNAP_FOR_REEXEC: 0
            TRUST_TEST_KEYS: false
            CORE_CHANNEL: stable
        systems:
            - ubuntu-22.04-64:
                image: snapd-spread/ubuntu-22.04-64
                workers: 8
            - ubuntu-24.04-64:
                image: snapd-spread/ubuntu-24.04-64
                workers: 8
            - ubuntu-25.10-64:
                image: snapd-spread/ubuntu-25.10-64
                workers: 8
            - ubuntu-26.04-64:
                image: snapd-spread/ubuntu-26.04-64
                workers: 8

    qemu-nested:
        memory: 4G
        type: qemu
        systems:
            - ubuntu-16.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-18.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-22.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-24.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-26.04-64:
                  username: ubuntu
                  password: ubuntu

    qemu:
        memory: 4G
        systems:
            - ubuntu-14.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-16.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-core-16-64:
                  image: ubuntu-16.04-64
                  username: ubuntu
                  password: ubuntu
            - ubuntu-core-18-64:
                  image: ubuntu-18.04-64
                  username: ubuntu
                  password: ubuntu
            - ubuntu-core-20-64:
                  image: ubuntu-20.04-64
                  username: ubuntu
                  password: ubuntu
                  bios: uefi
                  # TODO: remove once everyone switch to official spread
                  flags: [virtio]
            - ubuntu-core-22-64:
                  image: ubuntu-22.04-64
                  username: ubuntu
                  password: ubuntu
                  bios: uefi
                  # TODO: remove once everyone switch to official spread
                  flags: [virtio]
            - ubuntu-18.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-22.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-24.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-26.04-64:
                  username: ubuntu
                  password: ubuntu
            - debian-12-64:
                  username: debian
                  password: debian
            - debian-sid-64:
                  username: debian
                  password: debian
            - amazon-linux-2-64:
                  username: ec2-user
                  password: ec2-user
            - opensuse-tumbleweed-64:
                  username: opensuse
                  password: opensuse
    garden:
        type: adhoc
        allocate: |
            # Spread automatically injects /snap/bin to PATH. When we are
            # running from the image-garden snap then SPREAD_HOST_PATH is the
            # original path before such modifications were applied. Snap
            # applications cannot normally run /snap/bin/* entry-points
            # successfully so re-set PATH to the original value, as provided by
            # snapcraft.
            if [ -n "${SPREAD_HOST_PATH-}" ]; then
                PATH="${SPREAD_HOST_PATH}"
            fi
            # Spread does not expose any settings to ad-hoc backends.
            export QEMU_MEM_OPTION="-m 3072"
            export QEMU_SMP_OPTION="-smp 4"
            exec image-garden allocate "$(.image-garden/remap-name snapd-to-garden "$SPREAD_SYSTEM")"
        discard: |
            # Spread automatically injects /snap/bin to PATH. When we are
            # running from the image-garden snap then SPREAD_HOST_PATH is the
            # original path before such modifications were applied. Snap
            # applications cannot normally run /snap/bin/* entry-points
            # successfully so re-set PATH to the original value, as provided by
            # snapcraft.
            if [ -n "${SPREAD_HOST_PATH-}" ]; then
                PATH="${SPREAD_HOST_PATH}"
            fi
            # Stop the running qemu instance corresponding to the given address.
            exec image-garden discard "$SPREAD_SYSTEM_ADDRESS"
        systems:
            - ubuntu-core-16-64:
                  username: root
                  password: root
                  # Disable due to ENOSPC.
                  manual: true
            - ubuntu-core-18-64:
                  username: root
                  password: root
            - ubuntu-core-20-64:
                  username: root
                  password: root
            - ubuntu-core-22-64:
                  username: root
                  password: root
            - ubuntu-core-24-64:
                  username: root
                  password: root
            - ubuntu-core-26-64:
                  username: root
                  password: root
            - ubuntu-16.04-64:
                  username: root
                  password: root
            - ubuntu-18.04-64:
                  username: root
                  password: root
            - ubuntu-20.04-64:
                  username: root
                  password: root
            - ubuntu-22.04-64:
                  username: root
                  password: root
            - ubuntu-24.04-64:
                  username: root
                  password: root
            - ubuntu-25.10-64:
                  username: root
                  password: root
            - ubuntu-26.04-64:
                  username: root
                  password: root
            - debian-12-64:
                  username: root
                  password: root
            - debian-sid-64:
                  username: root
                  password: root
            - opensuse-16.0-64:
                  username: root
                  password: root
            - opensuse-tumbleweed-64:
                  username: root
                  password: root
            - opensuse-tumbleweed-selinux-64:
                  username: root
                  password: root
            - fedora-44-64:
                  username: root
                  password: root
            - centos-9-64:
                  username: root
                  password: root
            - arch-linux-64:
                  username: root
                  password: root
            - amazon-linux-2-64:
                  username: root
                  password: root
            - amazon-linux-2023-64:
                  username: root
                  password: root
    autopkgtest:
        type: adhoc
        allocate: |
            echo "Allocating ad-hoc $SPREAD_SYSTEM"
            if [ -z "${ADT_ARTIFACTS}" ]; then
                FATAL "adhoc only works inside autopkgtest"
                exit 1
            fi
            ADDRESS localhost:22
        discard: |
            echo "Discarding ad-hoc $SPREAD_SYSTEM"
        systems:
            # Trusty
            - ubuntu-14.04-amd64:
                  username: ubuntu
                  password: ubuntu
            # Xenial
            - ubuntu-16.04-amd64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-16.04-ppc64el:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-16.04-armhf:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-16.04-s390x:
                  username: ubuntu
                  password: ubuntu
            # Bionic
            - ubuntu-18.04-amd64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-18.04-ppc64el:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-18.04-armhf:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-18.04-s390x:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-18.04-arm64:
                  username: ubuntu
                  password: ubuntu
            # Focal
            - ubuntu-20.04-amd64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-ppc64el:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-armhf:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-s390x:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-arm64:
                  username: ubuntu
                  password: ubuntu
            # Jammy
            - ubuntu-22.04-amd64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-22.04-ppc64el:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-22.04-armhf:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-22.04-s390x:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-22.04-arm64:
                  username: ubuntu
                  password: ubuntu
            # noble
            - ubuntu-24.04-amd64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-24.04-ppc64el:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-24.04-armhf:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-24.04-s390x:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-24.04-arm64:
                  username: ubuntu
                  password: ubuntu
            # resolute
            - ubuntu-26.04-amd64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-26.04-ppc64el:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-26.04-armhf:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-26.04-s390x:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-26.04-arm64:
                  username: ubuntu
                  password: ubuntu
            # Debian Forky
            - debian-14-amd64:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            - debian-14-armhf:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            - debian-14-ppc64el:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            - debian-14-s390x:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            - debian-14-arm64:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            # Debian Trixie
            - debian-13-amd64:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            - debian-13-armhf:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            - debian-13-ppc64el:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            - debian-13-s390x:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            - debian-13-arm64:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            # Debian Sid
            - debian-sid-amd64:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            - debian-sid-i386:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            - debian-sid-armhf:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            - debian-sid-ppc64el:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            - debian-sid-s390x:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
            - debian-sid-arm64:
                  username: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'
                  password: '$(HOST: echo "$AUTOPKGTEST_NORMAL_USER")'

    external:
        type: adhoc
        environment:
            SPREAD_EXTERNAL_ADDRESS: '$(HOST: echo "${SPREAD_EXTERNAL_ADDRESS:-localhost:8022}")'
            TRUST_TEST_KEYS: "false"
        allocate: |
            ADDRESS $SPREAD_EXTERNAL_ADDRESS
        systems:
            - ubuntu-20.04-arm-32:
                username: root
                password: ubuntu
            - ubuntu-22.04-arm-32:
                username: root
                password: ubuntu
            - ubuntu-core-16-64:
                  username: external
                  password: ubuntu123
            - ubuntu-core-16-32:
                  username: external
                  password: ubuntu123
            - ubuntu-core-16-arm-64:
                  username: external
                  password: ubuntu123
            - ubuntu-core-16-arm-32:
                  username: external
                  password: ubuntu123
            - ubuntu-core-18-64:
                  username: external
                  password: ubuntu123
            - ubuntu-core-18-32:
                  username: external
                  password: ubuntu123
            - ubuntu-core-18-arm-64:
                  username: external
                  password: ubuntu123
            - ubuntu-core-18-arm-32:
                  username: external
                  password: ubuntu123
            - ubuntu-core-20-64:
                  username: external
                  password: ubuntu123
            - ubuntu-core-20-arm-64:
                  username: external
                  password: ubuntu123
            - ubuntu-core-20-arm-32:
                  username: external
                  password: ubuntu123
            - ubuntu-core-22-64:
                  username: external
                  password: ubuntu123
            - ubuntu-core-22-arm-64:
                  username: external
                  password: ubuntu123
            - ubuntu-core-22-arm-32:
                  username: external
                  password: ubuntu123
            - ubuntu-core-24-64:
                  username: external
                  password: ubuntu123
            - ubuntu-core-24-arm-64:
                  username: external
                  password: ubuntu123
            - ubuntu-core-26-64:
                  username: external
                  password: ubuntu123
            - ubuntu-core-26-arm-64:
                  username: external
                  password: ubuntu123

path: /home/gopath/src/github.com/snapcore/snapd

artifacts:
    # Populated with feature tag information when SPREAD_TAG_FEATURES is set.
    - feature-tags
    # Populated with feature tag information when SPREAD_SNAPD_STATE_LOCK_TRACE_THRESHOLD_MS is set.
    - state-locks

exclude:
    - .git
    - cmd/snap/snap
    - cmd/snapd/snapd
    - cmd/snapctl/snapctl
    - cmd/snap-exec/snap-exec
    - cmd/autom4te.cache
    - "*.o"
    - "*.a"
    - "*.pyc"
    - ./vendor
    - "*.snap"
    - ".image-garden"

debug-each: |
    if [ "$SPREAD_DEBUG_EACH" != 1 ]; then
        exit
    fi

    #shellcheck source=tests/lib/state.sh
    . "$TESTSLIB/state.sh"
    #shellcheck source=tests/lib/systems.sh
    . "$TESTSLIB/systems.sh"

    echo '# System information'
    cat /etc/os-release || true

    echo '# Kernel information'
    uname -a

    echo '# Apparmor information'
    apparmor_parser --version || true

    echo '# Go information'
    go version || true

    if tests.nested is-nested; then
        echo '# free space on the host'
        df -h || true
        du -ch /var/tmp/work-dir || true

        echo '# nested VM status'
        tests.nested vm status
        # filter out ^[ to ensure that the debug output gets not messed up
        tests.nested get serial-log | sed 's/\x1b//g'
        # add another echo in case the serial log is missing a newline
        echo

        remote.exec "sudo journalctl --no-pager -u snapd" || true

        exit
    fi

    echo "# definition of snapd.service"
    systemctl cat snapd.service || true
    echo "# status of snapd service"
    systemctl status snapd.service || true
    echo "# memory limits of snapd service that systemd uses"
    systemctl show snapd.service | grep -e MemoryMax= -e MemoryLimit= || true
    echo "# memory limits of snapd service that are actually set"
    cat /sys/fs/cgroup/memory/system.slice/snapd.service/memory.limit_in_bytes || true
    echo '# journal messages for snapd'
    "$TESTSTOOLS"/journal-state get-log -u snapd

    echo '# user sessions information'
    journalctl --user -u snapd.session-agent.service || true
    systemctl status --user snapd.session-agent  || true

    if ! is_cgroupv2; then
        # dump any information on device cgroup of current session
        cgroup_dev="$(awk -F: '/:devices:/ { print $3}' < /proc/self/cgroup || true)"
        if [ -n "$cgroup_dev" ]; then
            echo "# device cgroup $cgroup_dev"
            cat "/sys/fs/cgroup/devices/$cgroup_dev/devices.list" || true
        fi
    else
        echo "# snap confinement device filtering maps"
        ls -l /sys/fs/bpf/snap || true
    fi

    case "$SPREAD_SYSTEM" in
        fedora-*|centos-*|amazon-*)
            if [ -e "$RUNTIME_STATE_PATH/audit-stamp" ]; then
                ausearch -i -m AVC --checkpoint "$RUNTIME_STATE_PATH/audit-stamp" --start checkpoint || true
            else
                ausearch -i -m AVC || true
            fi
            (
                find /root/snap -printf '%Z\t%H/%P\n' || true
                find /home -regex '/home/[^/]*/snap\(/.*\)?' -printf '%Z\t%H/%P\n' || true
            ) | grep -v snappy_home_t || true
            find /var/snap -printf '%Z\t%H/%P\n' | grep -v snappy_var_t || true
            ;;
        opensuse-*)
            echo '# apparmor denials logged by auditd'
            ausearch -m AVC | grep DENIED || true
            ;;
        *)
            echo '# apparmor denials '
            dmesg --ctime | grep DENIED || true
            ;;
    esac
    echo '# seccomp denials (kills) '
    dmesg --ctime | grep type=1326 || true
    echo '# snap connections --all'
    snap connections --all || true
    echo '# free space'
    df -h || true
    echo '# mounts'
    # use ascii output to prevent travis from messing up the encoding
    findmnt --ascii -o+PROPAGATION || true
    echo "# processes"
    ps axl
    echo "# /var/lib/snapd"
    find /var/lib/snapd/ -not -path '/var/lib/snapd/snap/*' -ls || true
    echo '# system journal messages'
    journalctl -e

    # Keep it as the last step in debug-each
    echo '# tasks executed on system'
    if [ -f "$RUNTIME_STATE_PATH/runs" ]; then
        echo "EXECUTED_TESTS=$(cat $RUNTIME_STATE_PATH/runs)"
    fi
    # Add new line at the end of the debug output
    echo ''

rename:
    # Move content into a directory, so that deltas computed by repack benefit
    # from the content looking similar to codeload.github.com.
    - s,^,$DELTA_PREFIX,S

repack: |
    # For Linode, compute a delta based on a known git reference that can be
    # obtained directly from GitHub. There's nothing special about that reference,
    # other than it will often be in the local repository's history already.
    # The more recent the reference, the smaller the delta.
    if ! echo "$SPREAD_BACKENDS" | grep -e linode -e google; then
        cat <&3 >&4
    elif ! git show-ref "$DELTA_REF" > /dev/null; then
        cat <&3 >&4
    elif [ -n "${NO_DELTA-}" ]; then
        # delta has been disabled by the caller
        cat <&3 >&4
    else
        tmpdir="$(mktemp -d)"
        #shellcheck disable=SC2064
        trap "rm -rf delta-ref.tar current.delta repacked-current.tar $tmpdir" EXIT
        if [ "$REPACK_KEEP_VENDOR" = "n" ]; then
            tar -C "$tmpdir" -xvf - <&3
            rm -rf "$tmpdir"/$DELTA_PREFIX/vendor/*
            tar -C "$tmpdir" -c "$DELTA_PREFIX" --sort=name > repacked-current.tar
        else
            cat <&3 > repacked-current.tar
        fi
        git archive -o delta-ref.tar --format=tar --prefix="$DELTA_PREFIX" "$DELTA_REF"
        xdelta3 -S none -s delta-ref.tar repacked-current.tar > current.delta
        tar c current.delta >&4
    fi

kill-timeout: 30m
tasks-limit: 20000

prepare: |
    # NOTE: This part of the code needs to be in spread.yaml as it runs before
    # the rest of the source code (including the tests/lib directory) is
    # around. The purpose of this code is to fix some connectivity issues and
    # then apply the delta of the git repository.

    # apt update is hanging on security.ubuntu.com with IPv6, prefer IPv4 over IPv6
    cat <<EOF > gai.conf
    precedence  ::1/128       50
    precedence  ::/0          40
    precedence  2002::/16     30
    precedence ::/96          20
    precedence ::ffff:0:0/96 100
    EOF
    if ! mv gai.conf /etc/gai.conf; then
        echo "/etc/gai.conf is not writable, ubuntu-core system? apt update won't be affected in that case"
        rm -f gai.conf
    fi
    if command -v restorecon ; then
        # restore proper context otherwise SELinux may complain
        restorecon -v /etc/gai.conf
    fi

    # Note that os.query or any other tool cannot be used here before the current.delta file is unpacked
    if [[ "$SPREAD_SYSTEM" == fedora-* ]]; then
        dnf --refresh -y makecache

        # enable audit daemon
        systemctl enable --now auditd.service
    fi

    if [[ "$SPREAD_SYSTEM" == opensuse-* ]]; then
        # refresh metadatadata
        # Auto import gpg keys needed for could repository added to support google backend
        zypper --gpg-auto-import-keys ref
        # We seem to be hitting a flaky openSUSE mirror from time to time,
        # increase the number of download attempts libzypp will try to
        # workaround that.
        cat <<-EOF >> /etc/zypp/zypp.conf
    # added by spread tests
    download.max_silent_tries = 20
    EOF

        # Make sure docs are installed with the packages
        sed 's/rpm.install.excludedocs = yes/rpm.install.excludedocs = no/g' -i /etc/zypp/zypp.conf

        # TODO: enable enforcing mode
        if selinuxenabled; then
            setenforce Permissive
            # also make it permanent if the system were to reboot
            sed -i -e 's/^SELINUX=enforcing/SELINUX=permissive/g' /etc/selinux/config
        fi
    fi

    if [[ "$SPREAD_SYSTEM" == arch-* ]]; then
        # Possible that AppArmor was not started and is not enabled in the
        # image, do both now
        if systemctl show -p LoadState apparmor.service | MATCH 'LoadState=loaded' ; then
            if ! systemctl is-enabled apparmor.service; then
                systemctl enable apparmor.service
            fi
            systemctl start apparmor.service
        else
            exit 1
        fi
    fi

    if [[ "$SPREAD_SYSTEM" == debian-* ]]; then
        apt-get update && apt-get install -y eatmydata
    fi

    if [[ "$SPREAD_SYSTEM" == ubuntu-fips-* ]]; then
        if ! [ "$(cat /proc/sys/crypto/fips_enabled)" = "1" ]; then
            echo "FIPS not enabled"
            exit 1
        fi

        if apt-cache policy | NOMATCH golang-fips; then
            echo "golang-fips repo not found, the system is not properly configured for FIPS"
            exit 1
        fi
    fi

    # Enable EPEL see https://docs.fedoraproject.org/en-US/epel/
    case "$SPREAD_SYSTEM" in
        centos-9-*)
            dnf config-manager --set-enabled crb
            dnf install -y epel-release epel-next-release
            ;;
    esac

    case "$SPREAD_SYSTEM" in
        ubuntu-*|debian-*)
            # make sure unattended-upgrades does not get in the way
            if systemctl is-enabled unattended-upgrades.service; then
                systemctl stop unattended-upgrades.service
                systemctl mask unattended-upgrades.service
            fi
            ;;
    esac

    # Make sure ssh service is restarted after it is killed by spread (pkill -o -HUP sshd)
    # during the machine setup in google systems. For more details see lp:2011458
    if [[ "$SPREAD_BACKEND" =~ google ]] && [[ "$SPREAD_SYSTEM" == ubuntu-2* ]] && ! systemctl is-active ssh; then
        systemctl restart ssh
    fi

    # LSM correctness check
    case "$SPREAD_SYSTEM" in
        ubuntu-*|debian-*|arch-*)
            # Ubuntu, Debian, Arch use AppArmor
            aa-enabled
            ;;
        fedora-*|centos-*|opensuse-*-selinux-*)
            # Fedora, CentOS use SELinux
            # Variants openSUSE use selinux
            selinuxenabled
            ;;
        opensuse-*)
            # Non-SELinux variants of openSUSE use AppArmor
            aa-enabled
            ;;
        amazon-*-2023-*)
            # Amazon Linux 2023 uses SELinux
            selinuxenabled
            ;;
        amazon-*-2-*)
            # Amazon Linux 2 uses neither
            ! selinuxenabled
            ! aa-enabled
            ;;
        *)
            echo "unsupported system for LSM check"
            exit 1
            ;;
    esac

    # Unpack delta, or move content out of the prefixed directory (see rename and repack above).
    # (needs to be in spread.yaml directly because there's nothing else on the filesystem yet)
    if [ -f current.delta ]; then
        tf=$(mktemp)
        # NOTE: We can't use tests/lib/pkgdb.sh here as it doesn't exist at
        # this time when none of the test files is yet in place.
        case "$SPREAD_SYSTEM" in
            ubuntu-*|debian-*)
                apt-get update >& "$tf" || ( cat "$tf"; exit 1 )
                apt-get install -y xdelta3 curl eatmydata >& "$tf" || ( cat "$tf"; exit 1 )
                ;;
            amazon-linux-2023-*)
                echo "deltas are not supported on $SPREAD_SYSTEM, use NO_DELTA=1 when running spread"
                exit 1
                ;;
            amazon-*)
                yum install -y xdelta curl &> "$tf" || (cat "$tf"; exit 1)
                ;;
            fedora-*|centos-*)
                dnf install --refresh -y xdelta curl &> "$tf" || (cat "$tf"; exit 1)
                ;;
            opensuse-*)
                zypper -q --gpg-auto-import-keys refresh
                zypper -q install -y xdelta3 curl &> "$tf" || (cat "$tf"; exit 1)
                ;;
            arch-*)
                # there may be a libc upgrade which only -Syu handles;
                # ignore linux kernel as we would fail to detect it and handle
                # reboot; actual distro upgrade is done later in prepare.
                pacman -Syu --noconfirm xdelta3 curl --ignore linux &> "$tf" || (cat "$tf"; exit 1)
                ;;
        esac
        rm -f "$tf"
        curl -sS -o - "https://codeload.github.com/snapcore/snapd/tar.gz/$DELTA_REF" | gunzip > delta-ref.tar
        xdelta3 -q -c -d -s delta-ref.tar current.delta | tar x --strip-components=1
        rm -f delta-ref.tar current.delta
    elif [ -d "$DELTA_PREFIX" ]; then
        find "$DELTA_PREFIX" -mindepth 1 -maxdepth 1 -exec mv {} . \;
        rmdir "$DELTA_PREFIX"
    fi

    # Take the MATCH and REBOOT functions from spread and allow our shell
    # scripts to use them as shell commands. The replacements are real
    # executables in tests/lib/bin (which is on PATH) but they source
    # spread-funcs.sh written here, base on the definitions provided by SPREAD.
    # This ensures that 1) spread functions define the code 2) both MATCH and
    # REBOOT are executables and not functions, and can be called from any
    # context.
    type MATCH | tail -n +2 > "$TESTSLIB"/spread-funcs.sh
    unset MATCH
    type NOMATCH | tail -n +2 >> "$TESTSLIB"/spread-funcs.sh
    unset NOMATCH
    type REBOOT | tail -n +2 >> "$TESTSLIB"/spread-funcs.sh
    unset REBOOT

    # Copy external tools from the subtree to the "$TESTSLIB"/tools directory
    # The idea is to have a single directory with all the testing tools
    cp -f "$TESTSLIB"/external/snapd-testing-tools/tools/* "$TESTSTOOLS"
    cp -f "$TESTSLIB"/external/snapd-testing-tools/remote/* "$TESTSTOOLS"

    # ensure there are no broken snaps or the invariant test will fail later
    if command -v snap; then
        BROKEN="$(snap list --all | awk '/,?broken,?/ {print $1,$3}')"
        if [ -n "$BROKEN" ]; then
            echo "Test system has broken snaps:"
            snap list --all
            exit 1
        fi
    fi

    # native jq replacement, but still with some incompatibilies, see
    # https://github.com/itchyny/gojq
    # major differences:
    # - map keys are sorted by default
    # - with --yaml-input, can parse YAML
    if [ ! -e /tmp/gojq.tar.gz ]; then
        if os.query is-arm64; then
            "$TESTSTOOLS"/simpleget -o /tmp/gojq.tar.gz \
                https://github.com/itchyny/gojq/releases/download/v0.12.16/gojq_v0.12.16_linux_arm64.tar.gz
        elif os.query is-arm; then
            # upstream does not provide prebuilt binaries for armhf
            # built with: GOARCH=arm GOARM=7
            "$TESTSTOOLS"/simpleget -o /tmp/gojq.tar.gz \
                https://storage.googleapis.com/snapd-spread-tests/dependencies/gojq_v0.12.16_linux_armhf.tar.gz
        else
            # must be amd64
            "$TESTSTOOLS"/simpleget -o /tmp/gojq.tar.gz \
                https://github.com/itchyny/gojq/releases/download/v0.12.16/gojq_v0.12.16_linux_amd64.tar.gz
        fi
    fi

    (
        cd "$PROJECT_PATH/tests/bin"
        # only extract gojq, typically under a path:
        # -rwxr-xr-x runner/docker 4165784 2024-06-01 16:43 gojq_v0.12.16_linux_amd64/gojq
        tar -xvf /tmp/gojq.tar.gz --strip-components=1 --wildcards "*/gojq"
        test -x gojq
    )

    # NOTE: At this stage the source tree is available and no more special
    # considerations apply.
    "$TESTSLIB"/prepare-restore.sh --prepare-project
prepare-each: |
    "$TESTSLIB"/prepare-restore.sh --prepare-project-each
restore: |
    "$TESTSLIB"/prepare-restore.sh --restore-project
restore-each: |
    "$TESTSLIB"/prepare-restore.sh --restore-project-each
suites:
    tests/lib/tools/suite/:
        summary: Tests for tests/lib/tools tools
        backends: [google, qemu, garden]
        prepare-each: |
            tests.invariant set snap-mount-dir "$(os.paths snap-mount-dir)"
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each-minimal-no-snaps
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each-minimal-no-snaps
    tests/utils/cross-build/suite/:
        summary: Test building snapd in different architectures
        manual: true
        prepare-each: |
            tests.invariant set snap-mount-dir "$(os.paths snap-mount-dir)"
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each-minimal-no-snaps
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each-minimal-no-snaps
    # The essential tests designed to run inside the autopkgtest
    # environment on each platform. On autopkgtest we cannot run all tests
    # as this is very slow and we run into timeouts.
    #
    # These tests are executed on all other platforms as they
    # are designed to run on pristine systems
    tests/smoke/:
        summary: Essential system level tests for snapd
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite
    # All other tests run now and will heavily modify the system.
    tests/main/:
        summary: Full-system tests for snapd
        systems: [-ubuntu-secboot-*, -ubuntu-fips-*]
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite
        debug: |
            if [ "$SPREAD_DEBUG_EACH" = 1 ]; then
                systemctl status snapd.socket || true
            fi
    tests/core/:
        summary: Subset of Ubuntu Core specific tests
        systems: [ubuntu-core-*]
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite
    tests/completion/:
        summary: completion tests
        # ppc64el disabled because of https://bugs.launchpad.net/snappy/+bug/1655594
        systems:
            - -ubuntu-core-*
            - -ubuntu-*-ppc64el
            - -ubuntu-secboot-*
            - -ubuntu-fips-*
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite
        environment:
            _/plain: _
            _/plain_plusdirs: _
            _/funky: _
            _/files: _
            # dirs fails on indirection because of (mis)handling of trailing
            # slashes. This might be configuration-dependent.
            # _/dirs: _
            _/hosts: _
            _/hosts_n_dirs: _
            # twisted fails in travis (but not regular spread).
            #_/twisted: _
            _/func: _
            _/funkyfunc: _
            _/funcarg: _

    tests/regression/:
        summary: Regression tests for snapd
        systems: [-ubuntu-secboot-*, -ubuntu-fips-*]
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite

    tests/release/:
        summary: Tests for snapd during the release process
        systems: [ubuntu-1*, ubuntu-2*]
        environment:
            CORE_CHANNEL: stable
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        # This suite does not have restore steps due to the destructive
        # nature of its tests that will cause the vanila restore scripts to
        # fail. The tests in this suite are all meant to be run in isolation.

    tests/upgrade/:
        summary: Tests for snapd upgrade
        # Test cases are not yet ported to openSUSE that is why we keep
        # it disabled. A later PR will enable most tests and
        # drop the list of excluded systems.
        systems: [-ubuntu-core-*, -opensuse-*, -ubuntu-secboot-*]
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite
    tests/cross/:
        summary: Cross-compile tests
        systems: [ubuntu-24.04-64]
        prepare: |
            tests.invariant set snap-mount-dir "$(os.paths snap-mount-dir)"

    tests/unit/:
        summary: Suite to run unit tests (non-go and different go runtimes)
        # Test cases are not yet ported to Fedora/openSUSE/Arch that is why
        # we keep them disabled. A later PR will enable most tests and
        # drop the list of excluded systems.
        systems:
            [
                -ubuntu-core-*,
                -fedora-*,
                -opensuse-*,
                -arch-*,
                -amazon-*,
                -centos-*,
                -ubuntu-secboot-*,
                -ubuntu-fips-*,
            ]
        # unittests are run as part of the autopkgtest build already
        backends: [-autopkgtest]
        environment:
            # env vars required for coverage reporting from a spread task
            COVERMODE: '$(HOST: echo "$COVERMODE")'
        prepare: |
            #shellcheck source=tests/lib/prepare.sh
            . "$TESTSLIB"/prepare.sh
            prepare_classic
            tests.invariant set snap-mount-dir "$(os.paths snap-mount-dir)"
        prepare-each: |
            "$TESTSLIB"/reset.sh --reuse-core
            #shellcheck source=tests/lib/prepare.sh
            . "$TESTSLIB"/prepare.sh
            prepare_each_classic
        restore: |
            "$TESTSLIB"/reset.sh --store
            #shellcheck source=tests/lib/pkgdb.sh
            . "$TESTSLIB"/pkgdb.sh

            distro_purge_package snapd
            case "$SPREAD_SYSTEM" in
                arch-*)
                    # In Arch there is no separate snap-confine package.
                    ;;
                *)
                    distro_purge_package snap-confine ubuntu-core-launcher
                    ;;
            esac

    tests/nightly/:
        summary: Suite for nightly, expensive, tests
        manual: true
        # Test cases are not yet ported to Fedora/openSUSE/Arch/AMZN2 that is why
        # we keep them disabled. A later PR will enable most tests and
        # drop the list of excluded systems.
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite

    tests/perf/main/:
        summary: Performance and Load tests
        backends: [external]
        environment:
            CONNECTIONS_PERCENTAGE: '$(HOST: echo "${PERF_CONNECTIONS_PERCENTAGE:-100}")'
            DISCONNECT_INTERFACES: '$(HOST: echo "${PERF_DISCONNECT_INTERFACES:-true}")'
            CPU_LOAD: '$(HOST: echo "${PERF_CPU_LOAD:-}")'
            LOAD_DURATION: '$(HOST: echo "${PERF_LOAD_DURATION:-1200}")'
            NUM_PARALLEL: '$(HOST: echo "${PERF_NUM_PARALLEL:-10}")'
            NUM_SNAPS: '$(HOST: echo "${PERF_NUM_SNAPS:-100}")'
        manual: true
        warn-timeout: 10m
        kill-timeout: 60m
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
            if [ -n "$CPU_LOAD" ]; then
                # Start load generator and wait until it is active
                # It is generated as a systemd service to make it independent of snapd
                # and also to make sure it is not blocking spread scripts
                snap install --edge --devmode test-snapd-load-generator
                tests.systemd create-and-start-unit load-generator "/snap/bin/test-snapd-load-generator.run -d $LOAD_DURATION -l $CPU_LOAD"
                tests.systemd wait-for-service -n 5 --wait 1 --state active load-generator
            fi
        restore-each: |
            if [ -n "$CPU_LOAD" ]; then
                # Now we make sure all the load generator processes are killed before
                # the load-generator service is removed
                #shellcheck disable=SC2009
                PIDs="$(ps -ef | grep -e '^root.*python3.*load-generator' | awk '{print $2}')"
                # shellcheck disable=SC2086
                for pid in $PIDs; do
                    if ps -p "$pid" &>/dev/null; then
                        kill -9 "$pid"
                    fi
                done
                tests.systemd stop-unit --remove load-generator
            fi
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite

    tests/nested/manual/:
        summary: Tests for nested images controlled manually from the tests
        backends:
            - google-nested
            - qemu-nested
            - google-nested-arm
            - garden
            - openstack-ext
            - openstack-arm-ext
            - openstack-ext-dev
            - openstack-validation
        environment:
            NESTED_TYPE: "classic"
            # Enable kvm in the qemu command line
            NESTED_ENABLE_KVM: '$(HOST: echo "${NESTED_ENABLE_KVM:-true}")'
            # Enable tpm in the nested vm in case it is supported
            NESTED_ENABLE_TPM: '$(HOST: echo "${NESTED_ENABLE_TPM:-}")'
            # Enable secure boot in the nested vm in case it is supported
            NESTED_ENABLE_SECURE_BOOT: '$(HOST: echo "${NESTED_ENABLE_SECURE_BOOT:-}")'
            # Add snapd debug and log to serial console
            NESTED_SNAPD_DEBUG_TO_SERIAL: '$(HOST: echo "${NESTED_SNAPD_DEBUG_TO_SERIAL:-false}")'
            # Add any extra cmd line parameter to the nested vm
            NESTED_EXTRA_CMDLINE: '$(HOST: echo "${NESTED_EXTRA_CMDLINE:-}")'
            # Generic image name which has to be overwritten in the test
            NESTED_IMAGE_ID: "unset"
            # Configuration file used for remote tools
            REMOTE_CFG_FILE: "$PROJECT_PATH/remote.setup.cfg"
            # Define which tests to run in the nested vm in run-spread test
            NESTED_SPREAD_TESTS: '$(HOST: echo "${NESTED_SPREAD_TESTS:-}")'
            # Indicates that by default snapd pkg is built from current
            SNAPD_DEB_FROM_REPO: '$(HOST: echo "${SPREAD_SNAPD_DEB_FROM_REPO:-false}")'
            # Use the specified username or certificate for ssh connections to the nested vm
            NESTED_REMOTE_USER_NAME: '$(HOST: echo "${NESTED_REMOTE_USER_NAME:-user1}")'
            NESTED_REMOTE_USER_CERT: '$(HOST: echo "${NESTED_REMOTE_USER_CERT:-}")'
        manual: true
        warn-timeout: 10m
        kill-timeout: 60m
        prepare: |
            #shellcheck source=tests/lib/prepare.sh
            . "$TESTSLIB"/prepare.sh
            setup_system_proxy

            #shellcheck source=tests/lib/pkgdb.sh
            . "$TESTSLIB"/pkgdb.sh
            install_pkg_dependencies

            if os.query is-xenial; then
                # the new ubuntu-image expects mkfs to support -d option, which was not
                # supported yet by the version of mkfs that shipped with Ubuntu 16.04
                # also ubuntu-image binary is not prebuilt for arm instances
                snap install ubuntu-image --channel="$OLD_UBUNTU_IMAGE_SNAP_CHANNEL" --classic
            else
                #shellcheck source=tests/lib/image.sh
                . "$TESTSLIB"/image.sh
                get_ubuntu_image
            fi

            # Install the snapd built
            dpkg -i "$GOHOME"/snapd_*.deb

            # Configure the ssh connection to the test vm
            if [ -n "$NESTED_REMOTE_USER_CERT" ]; then
                remote.setup config --host localhost --port 8022 --user "$NESTED_REMOTE_USER_NAME" --cert "$NESTED_REMOTE_USER_CERT"
            else
                remote.setup config --host localhost --port 8022 --user "$NESTED_REMOTE_USER_NAME" --pass ubuntu
            fi

            # Build and install initramfs package
            if os.query is-ubuntu-ge 24.04; then
                #shellcheck source=tests/lib/core-initrd.sh
                . "$TESTSLIB"/core-initrd.sh
                build_and_install_initramfs_deb
            fi
            tests.invariant set snap-mount-dir "$(os.paths snap-mount-dir)"

        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
            tests.nested prepare
            if os.query is-xenial && ! command -v ubuntu-image >/dev/null; then
                # This is needed because the snap in removed during on restore-each
                snap install ubuntu-image --channel="$OLD_UBUNTU_IMAGE_SNAP_CHANNEL" --classic
            fi
        restore-each: |
            "$TESTSLIB"/collect-artifacts.sh features --after-nested-task
            tests.nested vm remove
            tests.nested restore
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            tests.nested restore

    tests/nested/classic/:
        summary: Tests for nested images
        backends:
            - google-nested
            - qemu-nested
            - google-nested-arm
            - openstack-ext
            - openstack-arm-ext
            - openstack-ext-dev
        environment:
            NESTED_TYPE: "classic"
            # Channel used to create the nested vm
            NESTED_ENABLE_KVM: '$(HOST: echo "${NESTED_ENABLE_KVM:-true}")'
            # Enable tpm in the nested vm in case it is supported
            NESTED_ENABLE_TPM: '$(HOST: echo "${NESTED_ENABLE_TPM:-false}")'
            # Enable secure boot in the nested vm in case it is supported
            NESTED_ENABLE_SECURE_BOOT: '$(HOST: echo "${NESTED_ENABLE_SECURE_BOOT:-false}")'
            # Configuration file used for remote tools
            REMOTE_CFG_FILE: "$PROJECT_PATH/remote.setup.cfg"
            # Indicates that by default snapd pkg is built from current
            SNAPD_DEB_FROM_REPO: '$(HOST: echo "${SPREAD_SNAPD_DEB_FROM_REPO:-false}")'
            # Use the specified username or certificate for ssh connections to the nested vm
            NESTED_REMOTE_USER_NAME: '$(HOST: echo "${NESTED_REMOTE_USER_NAME:-user1}")'
            NESTED_REMOTE_USER_CERT: '$(HOST: echo "${NESTED_REMOTE_USER_CERT:-}")'
        manual: true
        prepare: |
            #shellcheck source=tests/lib/prepare.sh
            . "$TESTSLIB"/prepare.sh
            setup_system_proxy

            #shellcheck source=tests/lib/pkgdb.sh
            . "$TESTSLIB"/pkgdb.sh
            install_pkg_dependencies

            if os.query is-xenial; then
                # the new ubuntu-image expects mkfs to support -d option, which was not
                # supported yet by the version of mkfs that shipped with Ubuntu 16.04
                # also ubuntu-image binary is not prebuilt for arm instances
                snap install ubuntu-image --channel="$OLD_UBUNTU_IMAGE_SNAP_CHANNEL" --classic
            elif os.query is-arm; then
                snap install ubuntu-image --channel="$UBUNTU_IMAGE_SNAP_CHANNEL" --classic
            else
                #shellcheck source=tests/lib/image.sh
                . "$TESTSLIB"/image.sh
                get_ubuntu_image
            fi

            # Install the snapd built
            dpkg -i "$GOHOME"/snapd_*.deb

            # Configure the ssh connection to the test vm
            # Configure the ssh connection to the test vm
            if [ -n "$NESTED_REMOTE_USER_CERT" ]; then
                remote.setup config --host localhost --port 8022 --user "$NESTED_REMOTE_USER_NAME" --cert "$NESTED_REMOTE_USER_CERT"
            else
                remote.setup config --host localhost --port 8022 --user "$NESTED_REMOTE_USER_NAME" --pass ubuntu
            fi

            tests.nested prepare
            tests.nested build-image classic
            tests.invariant set snap-mount-dir "$(os.paths snap-mount-dir)"
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
            tests.nested create-vm classic
        restore-each: |
            "$TESTSLIB"/collect-artifacts.sh features --after-nested-task
            tests.nested vm remove
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            tests.nested restore

    tests/nested/core/:
        summary: Tests for nested images
        backends:
            - google-nested
            - qemu-nested
            - google-nested-arm
            - openstack-ext
            - openstack-arm-ext
            - openstack-ext-dev
        environment:
            NESTED_TYPE: "core"
            # Enable kvm in the qemu command line
            NESTED_ENABLE_KVM: '$(HOST: echo "${NESTED_ENABLE_KVM:-true}")'
            # Enable tpm in the nested vm in case it is supported
            NESTED_ENABLE_TPM: '$(HOST: echo "${NESTED_ENABLE_TPM:-}")'
            # Enable secure boot in the nested vm in case it is supported
            NESTED_ENABLE_SECURE_BOOT: '$(HOST: echo "${NESTED_ENABLE_SECURE_BOOT:-}")'
            # Add snapd debug and log to serial console
            NESTED_SNAPD_DEBUG_TO_SERIAL: '$(HOST: echo "${NESTED_SNAPD_DEBUG_TO_SERIAL:-false}")'
            # Add any extra cmd line parameter to the nested vm
            NESTED_EXTRA_CMDLINE: '$(HOST: echo "${NESTED_EXTRA_CMDLINE:-}")'
            # Configuration file used for remote tools
            REMOTE_CFG_FILE: "$PROJECT_PATH/remote.setup.cfg"
            # Indicates that by default snapd pkg is built from current
            SNAPD_DEB_FROM_REPO: '$(HOST: echo "${SPREAD_SNAPD_DEB_FROM_REPO:-false}")'
            # Use the specified username or certificate for ssh connections to the nested vm
            NESTED_REMOTE_USER_NAME: '$(HOST: echo "${NESTED_REMOTE_USER_NAME:-user1}")'
            NESTED_REMOTE_USER_CERT: '$(HOST: echo "${NESTED_REMOTE_USER_CERT:-}")'
        manual: true
        prepare: |
            #shellcheck source=tests/lib/prepare.sh
            . "$TESTSLIB"/prepare.sh
            setup_system_proxy

            #shellcheck source=tests/lib/pkgdb.sh
            . "$TESTSLIB"/pkgdb.sh
            install_pkg_dependencies

            if os.query is-xenial; then
                # the new ubuntu-image expects mkfs to support -d option, which was not
                # supported yet by the version of mkfs that shipped with Ubuntu 16.04
                # also ubuntu-image binary is not prebuilt for arm instances
                snap install ubuntu-image --channel="$OLD_UBUNTU_IMAGE_SNAP_CHANNEL" --classic
            elif os.query is-arm; then
                snap install ubuntu-image --channel="$UBUNTU_IMAGE_SNAP_CHANNEL" --classic
            else
                #shellcheck source=tests/lib/image.sh
                . "$TESTSLIB"/image.sh
                get_ubuntu_image
            fi

            # Install the snapd built
            dpkg -i "$GOHOME"/snapd_*.deb

            # Configure the ssh connection to the test vm
            if [ -n "$NESTED_REMOTE_USER_CERT" ]; then
                remote.setup config --host localhost --port 8022 --user "$NESTED_REMOTE_USER_NAME" --cert "$NESTED_REMOTE_USER_CERT"
            else
                remote.setup config --host localhost --port 8022 --user "$NESTED_REMOTE_USER_NAME" --pass ubuntu
            fi

            # Build and install initramfs package
            if os.query is-ubuntu-ge 24.04; then
                #shellcheck source=tests/lib/core-initrd.sh
                . "$TESTSLIB"/core-initrd.sh
                build_and_install_initramfs_deb
            fi

            tests.nested prepare
            tests.nested build-image core
            tests.invariant set snap-mount-dir "$(os.paths snap-mount-dir)"
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
            tests.nested create-vm core
        restore-each: |
            "$TESTSLIB"/collect-artifacts.sh features --after-nested-task
            tests.nested vm remove
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            tests.nested restore

    tests/perf/nested/:
        summary: Performance and Load tests preparation suite
        backends:
            - google-nested
            - qemu-nested
            - google-nested-arm
            - openstack-ext
            - openstack-arm-ext
            - openstack-ext-dev
        environment:
            NESTED_TYPE: "core"
            # Enable kvm in the qemu command line
            NESTED_ENABLE_KVM: '$(HOST: echo "${NESTED_ENABLE_KVM:-true}")'
            # Enable tpm in the nested vm in case it is supported
            NESTED_ENABLE_TPM: '$(HOST: echo "${NESTED_ENABLE_TPM:-true}")'
            # Enable secure boot in the nested vm in case it is supported
            NESTED_ENABLE_SECURE_BOOT: '$(HOST: echo "${NESTED_ENABLE_SECURE_BOOT:-true}")'
            # Add snapd debug and log to serial console
            NESTED_SNAPD_DEBUG_TO_SERIAL: '$(HOST: echo "${NESTED_SNAPD_DEBUG_TO_SERIAL:-false}")'
            # Add any extra cmd line parameter to the nested vm
            NESTED_EXTRA_CMDLINE: '$(HOST: echo "${NESTED_EXTRA_CMDLINE:-}")'
            # Configuration file used for remote tools
            REMOTE_CFG_FILE: "$PROJECT_PATH/remote.setup.cfg"
            # Indicates the number of cpus to use in the nested vm
            NESTED_CPUS: '$(HOST: echo "${NESTED_CPUS:-2}")'
            # Indicates the amount of memory in MB to use in the nested vm
            NESTED_MEM: '$(HOST: echo "${NESTED_MEM:-4096}")'
            SPREAD_URL: https://storage.googleapis.com/snapd-spread-tests/spread/spread-amd64.tar.gz
            # Indicates that by default snapd pkg is built from current
            SNAPD_DEB_FROM_REPO: '$(HOST: echo "${SPREAD_SNAPD_DEB_FROM_REPO:-false}")'
            # Use the specified username or certificate for ssh connections to the nested vm
            NESTED_REMOTE_USER_NAME: '$(HOST: echo "${NESTED_REMOTE_USER_NAME:-user1}")'
            NESTED_REMOTE_USER_CERT: '$(HOST: echo "${NESTED_REMOTE_USER_CERT:-}")'
        manual: true
        warn-timeout: 10m
        kill-timeout: 60m
        prepare: |
            #shellcheck source=tests/lib/prepare.sh
            . "$TESTSLIB"/prepare.sh
            setup_system_proxy

            #shellcheck source=tests/lib/pkgdb.sh
            . "$TESTSLIB"/pkgdb.sh
            install_pkg_dependencies

            if os.query is-xenial; then
                # the new ubuntu-image expects mkfs to support -d option, which was not
                # supported yet by the version of mkfs that shipped with Ubuntu 16.04
                # also ubuntu-image binary is not prebuilt for arm instances
                snap install ubuntu-image --channel="$OLD_UBUNTU_IMAGE_SNAP_CHANNEL" --classic
            elif os.query is-arm; then
                snap install ubuntu-image --channel="$UBUNTU_IMAGE_SNAP_CHANNEL" --classic
            else
                #shellcheck source=tests/lib/image.sh
                . "$TESTSLIB"/image.sh
                get_ubuntu_image
            fi

            # Install the snapd built
            dpkg -i "$GOHOME"/snapd_*.deb

            # Configure the ssh connection to the test vm
            if [ -n "$NESTED_REMOTE_USER_CERT" ]; then
                remote.setup config --host localhost --port 8022 --user "$NESTED_REMOTE_USER_NAME" --cert "$NESTED_REMOTE_USER_CERT"
            else
                remote.setup config --host localhost --port 8022 --user "$NESTED_REMOTE_USER_NAME" --pass ubuntu
            fi

            tests.nested prepare
            tests.nested build-image core
            tests.invariant set snap-mount-dir "$(os.paths snap-mount-dir)"
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
            tests.nested create-vm core --param-cpus "$NESTED_CPUS" --param-mem "$NESTED_MEM"

        restore-each: |
            tests.nested vm remove
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each

        restore: |
            tests.nested restore

    # FIPS test suite running with snapd snap
    tests/fips-snap/:
        summary: FIPS enabled snapd snap test suite
        systems:
            - ubuntu-fips-*
        environment:
            SNAP_REEXEC: "1"
            SNAPD_DEB_FROM_REPO: "true"
            FIPS_ARTIFACT: "snap"
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite
        debug: |
            if [ "$SPREAD_DEBUG_EACH" = 1 ]; then
                systemctl status snapd.socket || true
            fi

    # FIPS test suite running with snapd DEB
    tests/fips-deb/:
        summary: FIPS enabled snapd deb test suite
        systems:
            - ubuntu-fips-*
        environment:
            # disable reexec so that we use the deb and and explicitly build the
            # deb from the source tree
            SNAP_REEXEC: "0"
            SNAPD_DEB_FROM_REPO: "false"
            FIPS_ARTIFACT: "deb"
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite
        debug: |
            if [ "$SPREAD_DEBUG_EACH" = 1 ]; then
                systemctl status snapd.socket || true
            fi

    # a subset of the main test suite but for cross distro and with reexec enabled
    tests/cross-distro-reexec/:
        summary: Cross-distro reeexec
        systems:
            - arch-linux-*
            - opensuse-*
            - amazon-linux-2-*
            - amazon-linux-2023-*
            - fedora-*
            - centos-*
        environment:
            # enable snap rexec
            SNAP_REEXEC: "1"
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
            SNAP_MOUNT_DIR="$(os.paths snap-mount-dir)"
            case "$SNAP_MOUNT_DIR" in
                /var/lib/snapd/snap)
                    ln -sf "$SNAP_MOUNT_DIR" /snap
                    tests.cleanup defer rm -f /snap
                    ;;
            esac

            systemctl restart snapd.service

            # assert state of reexec
            pmap -p "$(pidof snapd)" | MATCH "$SNAP_MOUNT_DIR/snapd/.*/usr/lib/snapd/snapd"
            snap debug execution snap | MATCH 'is-reexecd: true'
            # assert internal apparmor?
            if command -v aa-enabled 2>/dev/null && aa-enabled ; then
                snap debug execution apparmor | MATCH 'internal: true'
            fi

        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite
        debug: |
            if [ "$SPREAD_DEBUG_EACH" = 1 ]; then
                systemctl status snapd.socket || true
            fi

# vim:ts=4:sw=4:et