From 11a2e5949b9c156583d754c4e11016e940e13afd Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 4 Mar 2026 14:02:32 +0800 Subject: [PATCH 01/61] Add ubuntu 26.04 (resolute) support --- .github/workflows/integration_test_app.yaml | 8 +++++++- app/src/github_runner_image_builder/config.py | 7 ++++++- app/tests/integration/helpers.py | 2 +- charmcraft.yaml | 2 +- src/state.py | 2 +- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration_test_app.yaml b/.github/workflows/integration_test_app.yaml index 089ee6a5..c81933f3 100644 --- a/.github/workflows/integration_test_app.yaml +++ b/.github/workflows/integration_test_app.yaml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - image: [focal, jammy, noble] + image: [focal, jammy, noble, resolute] arch: [amd64, arm64, s390x, ppc64le] exclude: - image: focal @@ -27,6 +27,12 @@ jobs: arch: ppc64le - image: jammy arch: s390x + - image: resolute + arch: arm64 + - image: resolute + arch: ppc64le + - image: resolute + arch: s390x steps: - uses: actions/checkout@v6.0.2 - uses: canonical/setup-lxd@v0.1.3 diff --git a/app/src/github_runner_image_builder/config.py b/app/src/github_runner_image_builder/config.py index 23802f0d..579fe295 100644 --- a/app/src/github_runner_image_builder/config.py +++ b/app/src/github_runner_image_builder/config.py @@ -52,14 +52,16 @@ class BaseImage(str, Enum): FOCAL: The focal ubuntu LTS image. JAMMY: The jammy ubuntu LTS image. NOBLE: The noble ubuntu LTS image. + RESOLUTE: The resolute ubuntu LTS image. """ FOCAL = "focal" JAMMY = "jammy" NOBLE = "noble" + RESOLUTE = "resolute" @classmethod - def get_version(cls, base: "BaseImage") -> Literal["20.04", "22.04", "24.04"]: + def get_version(cls, base: "BaseImage") -> Literal["20.04", "22.04", "24.04", "26.04"]: """Change the codename to version tag. Args: @@ -75,6 +77,8 @@ def get_version(cls, base: "BaseImage") -> Literal["20.04", "22.04", "24.04"]: return "22.04" case BaseImage.NOBLE: return "24.04" + case BaseImage.RESOLUTE: + return "26.04" @classmethod def from_str(cls, tag_or_name: str) -> "BaseImage": @@ -95,6 +99,7 @@ def from_str(cls, tag_or_name: str) -> "BaseImage": "20.04": BaseImage.FOCAL.value, "22.04": BaseImage.JAMMY.value, "24.04": BaseImage.NOBLE.value, + "26.04": BaseImage.RESOLUTE.value, } BASE_CHOICES = tuple( itertools.chain.from_iterable((tag, name) for (tag, name) in LTS_IMAGE_VERSION_TAG_MAP.items()) diff --git a/app/tests/integration/helpers.py b/app/tests/integration/helpers.py index 38f11a49..7fe66fc6 100644 --- a/app/tests/integration/helpers.py +++ b/app/tests/integration/helpers.py @@ -114,7 +114,7 @@ def create_lxd_vm_image( return lxd_image -IMAGE_TO_TAG = {"focal": "20.04", "jammy": "22.04", "noble": "24.04"} +IMAGE_TO_TAG = {"focal": "20.04", "jammy": "22.04", "noble": "24.04", "resolute": "26.04"} def _create_metadata_tar_gz(image: str, tmp_path: Path) -> Path: diff --git a/charmcraft.yaml b/charmcraft.yaml index 24f29948..56824386 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -82,7 +82,7 @@ config: description: | The base ubuntu OS image to use for the runners. Codename (e.g. "noble") or version tag (e.g. 24.04) is supported as input. Currently only supports LTS versions of focal and - higher, i.e. focal, jammy, noble. + higher, i.e. focal, jammy, noble, resolute. build-flavor: type: string default: "" diff --git a/src/state.py b/src/state.py index b2cc2aeb..9ef862d9 100644 --- a/src/state.py +++ b/src/state.py @@ -21,7 +21,7 @@ ARCHITECTURES_PPC64LE = {"ppc64le", "ppc64el"} ARCHITECTURES_X86 = {"x86_64", "amd64", "x64"} CLOUD_NAME = "builder" -LTS_IMAGE_VERSION_TAG_MAP = {"20.04": "focal", "22.04": "jammy", "24.04": "noble"} +LTS_IMAGE_VERSION_TAG_MAP = {"20.04": "focal", "22.04": "jammy", "24.04": "noble", "26.04": "resolute"} ARCHITECTURE_CONFIG_NAME = "architecture" BASE_IMAGE_CONFIG_NAME = "base-image" From 36db9a346ef40a30b8d2be3a8b4ea486c4fb78d4 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 5 Mar 2026 09:32:25 +0800 Subject: [PATCH 02/61] Add downloading of resolute image --- .../openstack_builder.py | 13 ++++++++++++- src/charm.py | 6 ++++-- src/state.py | 7 ++++++- tests/unit/test_builder.py | 10 ++++++++-- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index c94743c4..2c2a8d84 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -145,6 +145,10 @@ def initialize(arch: Arch, cloud_name: str, prefix: str) -> None: noble_image_path = cloud_image.download_and_validate_image( arch=arch, base_image=BaseImage.NOBLE, release_date=noble_release_date ) + logger.info("Downloading Resolute image.") + resolute_image_path = cloud_image.download_and_validate_image( + arch=arch, base_image=BaseImage.RESOLUTE + ) logger.info("Uploading Focal image.") store.upload_image( arch=arch, @@ -169,7 +173,14 @@ def initialize(arch: Arch, cloud_name: str, prefix: str) -> None: image_path=noble_image_path, keep_revisions=1, ) - + logger.info("Uploading Resolute image.") + store.upload_image( + arch=arch, + cloud_name=cloud_name, + image_name=_get_base_image_name(arch=arch, base=BaseImage.RESOLUTE, prefix=prefix), + image_path=resolute_image_path, + keep_revisions=1, + ) with openstack.connect(cloud=cloud_name) as conn: _create_keypair(conn=conn, prefix=prefix) logger.info("Creating security group %s.", SHARED_SECURITY_GROUP_NAME) diff --git a/src/charm.py b/src/charm.py index cdecae48..242a9b30 100755 --- a/src/charm.py +++ b/src/charm.py @@ -212,7 +212,8 @@ def _setup_builder(self) -> None: def _setup_logrotate(self) -> None: """Set up the log rotation for image-builder application.""" APP_LOGROTATE_CONFIG_PATH.write_text( - dedent(f"""\ + dedent( + f"""\ {str(LOG_FILE_PATH.absolute())} {{ weekly rotate 3 @@ -220,7 +221,8 @@ def _setup_logrotate(self) -> None: delaycompress missingok }} - """), + """ + ), encoding="utf-8", ) try: diff --git a/src/state.py b/src/state.py index 9ef862d9..0cc369cd 100644 --- a/src/state.py +++ b/src/state.py @@ -21,7 +21,12 @@ ARCHITECTURES_PPC64LE = {"ppc64le", "ppc64el"} ARCHITECTURES_X86 = {"x86_64", "amd64", "x64"} CLOUD_NAME = "builder" -LTS_IMAGE_VERSION_TAG_MAP = {"20.04": "focal", "22.04": "jammy", "24.04": "noble", "26.04": "resolute"} +LTS_IMAGE_VERSION_TAG_MAP = { + "20.04": "focal", + "22.04": "jammy", + "24.04": "noble", + "26.04": "resolute", +} ARCHITECTURE_CONFIG_NAME = "architecture" BASE_IMAGE_CONFIG_NAME = "base-image" diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 57e16e15..813ab8d4 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -257,7 +257,9 @@ def test_install_clouds_yaml_not_exists(monkeypatch: pytest.MonkeyPatch, tmp_pat ) contents = test_path.read_text(encoding="utf-8") - assert contents == f"""clouds: + assert ( + contents + == f"""clouds: test: auth: auth_url: test-url @@ -267,6 +269,7 @@ def test_install_clouds_yaml_not_exists(monkeypatch: pytest.MonkeyPatch, tmp_pat user_domain_name: test_domain username: test-user """ + ) def test_install_clouds_yaml_unchanged(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): @@ -299,7 +302,9 @@ def test_install_clouds_yaml_unchanged(monkeypatch: pytest.MonkeyPatch, tmp_path builder.install_clouds_yaml(cloud_config=test_config) contents = test_path.read_text(encoding="utf-8") - assert contents == f"""clouds: + assert ( + contents + == f"""clouds: test: auth: auth_url: test-url @@ -309,6 +314,7 @@ def test_install_clouds_yaml_unchanged(monkeypatch: pytest.MonkeyPatch, tmp_path user_domain_name: test_domain username: test-user """ + ) @pytest.mark.parametrize( From 7b57328df4cace6a1228cc4a9fefef8a767006f9 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 5 Mar 2026 15:21:35 +0800 Subject: [PATCH 03/61] Disable HWE kernel for resolute --- .../templates/cloud-init.sh.j2 | 7 +++++-- app/tests/integration/commands.py | 3 ++- app/tests/unit/test_config.py | 1 + app/tests/unit/test_openstack_builder.py | 7 +++++-- src/charm.py | 6 ++---- tests/unit/test_builder.py | 10 ++-------- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 index f13fe4b4..762b8645 100644 --- a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 +++ b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 @@ -72,8 +72,11 @@ function install_apt_packages() { DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get upgrade -y echo "Installing apt packages $packages" DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install -y --no-install-recommends ${packages} - echo "Installing linux-generic-hwe-${hwe_version}" - DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install -y --install-recommends linux-generic-hwe-${hwe_version} + # Exclude resolute from HWE kernel test since there is no HWE kernel for it. + if [ $RELEASE != "resolute" ]; then + echo "Installing linux-generic-hwe-${hwe_version}" + DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install -y --install-recommends linux-generic-hwe-${hwe_version} + fi } function disable_unattended_upgrades() { diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 112f9b82..d5cee918 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -59,9 +59,10 @@ class Commands: Commands( name="test sctp support", command="sudo apt-get install lksctp-tools -yq && checksctp" ), + # Exclude 26.04 from HWE kernel test since there is no HWE kernel for it. Commands( name="test that HWE kernel is installed", - command="uname -a | " + command="lsb_release -r | grep 26.04 || uname -a | " "grep $(dpkg -l | grep linux-generic-hwe | awk '{print $3}' | cut -d'.' -f1-3)", ), Commands( diff --git a/app/tests/unit/test_config.py b/app/tests/unit/test_config.py index d3c646a7..dac4243d 100644 --- a/app/tests/unit/test_config.py +++ b/app/tests/unit/test_config.py @@ -75,6 +75,7 @@ def test_base_image(image: str, expected_base_image: BaseImage): pytest.param(BaseImage.FOCAL, "20.04", id="focal"), pytest.param(BaseImage.JAMMY, "22.04", id="jammy"), pytest.param(BaseImage.NOBLE, "24.04", id="noble"), + pytest.param(BaseImage.RESOLUTE, "26.04", id="resolute"), pytest.param(None, None, id="None"), ], ) diff --git a/app/tests/unit/test_openstack_builder.py b/app/tests/unit/test_openstack_builder.py index 2a1278e2..7fc96b52 100644 --- a/app/tests/unit/test_openstack_builder.py +++ b/app/tests/unit/test_openstack_builder.py @@ -783,8 +783,11 @@ def test__generate_cloud_init_script( DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get upgrade -y echo "Installing apt packages $packages" DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install -y --no-install-recommends ${{packages}} - echo "Installing linux-generic-hwe-${{hwe_version}}" - DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install -y --install-recommends linux-generic-hwe-${{hwe_version}} + # Exclude resolute from HWE kernel test since there is no HWE kernel for it. + if [ $RELEASE != "resolute" ]; then + echo "Installing linux-generic-hwe-${{hwe_version}}" + DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install -y --install-recommends linux-generic-hwe-${{hwe_version}} + fi }} function disable_unattended_upgrades() {{ diff --git a/src/charm.py b/src/charm.py index 242a9b30..cdecae48 100755 --- a/src/charm.py +++ b/src/charm.py @@ -212,8 +212,7 @@ def _setup_builder(self) -> None: def _setup_logrotate(self) -> None: """Set up the log rotation for image-builder application.""" APP_LOGROTATE_CONFIG_PATH.write_text( - dedent( - f"""\ + dedent(f"""\ {str(LOG_FILE_PATH.absolute())} {{ weekly rotate 3 @@ -221,8 +220,7 @@ def _setup_logrotate(self) -> None: delaycompress missingok }} - """ - ), + """), encoding="utf-8", ) try: diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index 813ab8d4..57e16e15 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -257,9 +257,7 @@ def test_install_clouds_yaml_not_exists(monkeypatch: pytest.MonkeyPatch, tmp_pat ) contents = test_path.read_text(encoding="utf-8") - assert ( - contents - == f"""clouds: + assert contents == f"""clouds: test: auth: auth_url: test-url @@ -269,7 +267,6 @@ def test_install_clouds_yaml_not_exists(monkeypatch: pytest.MonkeyPatch, tmp_pat user_domain_name: test_domain username: test-user """ - ) def test_install_clouds_yaml_unchanged(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): @@ -302,9 +299,7 @@ def test_install_clouds_yaml_unchanged(monkeypatch: pytest.MonkeyPatch, tmp_path builder.install_clouds_yaml(cloud_config=test_config) contents = test_path.read_text(encoding="utf-8") - assert ( - contents - == f"""clouds: + assert contents == f"""clouds: test: auth: auth_url: test-url @@ -314,7 +309,6 @@ def test_install_clouds_yaml_unchanged(monkeypatch: pytest.MonkeyPatch, tmp_path user_domain_name: test_domain username: test-user """ - ) @pytest.mark.parametrize( From 7ee138d4172fffb597ba9adff859a5ff28908ca4 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Fri, 6 Mar 2026 08:42:23 +0800 Subject: [PATCH 04/61] Use tmate for debugging --- .github/workflows/integration_test_app.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration_test_app.yaml b/.github/workflows/integration_test_app.yaml index c81933f3..c52b05dd 100644 --- a/.github/workflows/integration_test_app.yaml +++ b/.github/workflows/integration_test_app.yaml @@ -73,6 +73,9 @@ jobs: run: | tox -e integration -- --arch ppc64le --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS_APP_PPC64LE }} working-directory: app + # DEBUG + - name: Setup tmate session + uses: canonical/action-tmate@main required_status_checks: name: Required Integration Test For Application Status Checks runs-on: ubuntu-latest From 75684f272b46dbc16ad5620dc5ed1b830c6c260d Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Fri, 6 Mar 2026 09:31:58 +0800 Subject: [PATCH 05/61] Print out debug info --- .github/workflows/integration_test_app.yaml | 3 --- app/tests/integration/commands.py | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration_test_app.yaml b/.github/workflows/integration_test_app.yaml index c52b05dd..c81933f3 100644 --- a/.github/workflows/integration_test_app.yaml +++ b/.github/workflows/integration_test_app.yaml @@ -73,9 +73,6 @@ jobs: run: | tox -e integration -- --arch ppc64le --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS_APP_PPC64LE }} working-directory: app - # DEBUG - - name: Setup tmate session - uses: canonical/action-tmate@main required_status_checks: name: Required Integration Test For Application Status Checks runs-on: ubuntu-latest diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index d5cee918..3489b353 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -69,6 +69,10 @@ class Commands: name="test network congestion policy(fq)", command="sudo sysctl -a | grep 'net.core.default_qdisc = fq'", ), + Commands( + name="DEBUG", + command="sudo sysctl -a", + ), Commands( name="test network congestion policy", command="sudo sysctl -a | grep 'net.ipv4.tcp_congestion_control = bbr'", From 350dc4777beab1b087e915337dc334dd6c1cbe9a Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Fri, 6 Mar 2026 14:05:05 +0800 Subject: [PATCH 06/61] Remove check for bbr as TCP conestion_control for 26.04 --- app/tests/integration/commands.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 3489b353..fb5f9482 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -69,13 +69,9 @@ class Commands: name="test network congestion policy(fq)", command="sudo sysctl -a | grep 'net.core.default_qdisc = fq'", ), - Commands( - name="DEBUG", - command="sudo sysctl -a", - ), Commands( name="test network congestion policy", - command="sudo sysctl -a | grep 'net.ipv4.tcp_congestion_control = bbr'", + command="lsb_release -r | grep 26.04 || sudo sysctl -a | grep 'net.ipv4.tcp_congestion_control = bbr'", ), Commands( name="test external script", From 93bd704b5159498f7f1b03c0584470e82027b8e9 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Fri, 6 Mar 2026 14:26:37 +0800 Subject: [PATCH 07/61] Enable bbr TCP congestion control in cloud init --- app/src/github_runner_image_builder/templates/cloud-init.sh.j2 | 1 + app/tests/integration/commands.py | 2 +- app/tests/unit/test_openstack_builder.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 index 762b8645..6e7000a8 100644 --- a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 +++ b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 @@ -89,6 +89,7 @@ function disable_unattended_upgrades() { } function enable_network_fair_queuing_congestion() { + /usr/sbin/modprobe tcp_bbr /usr/bin/cat < Date: Fri, 6 Mar 2026 15:31:11 +0800 Subject: [PATCH 08/61] Debug --- app/tests/integration/commands.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index d5cee918..2f7f9994 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -69,6 +69,10 @@ class Commands: name="test network congestion policy(fq)", command="sudo sysctl -a | grep 'net.core.default_qdisc = fq'", ), + Commands( + name="Debug", + command="sudo sysctl -a", + ), Commands( name="test network congestion policy", command="sudo sysctl -a | grep 'net.ipv4.tcp_congestion_control = bbr'", From 8ecf281cd0a367ad393a9d66038a79be429110e8 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Fri, 6 Mar 2026 16:08:29 +0800 Subject: [PATCH 09/61] Remove tcp_bbr support for 26.04 --- .../github_runner_image_builder/templates/cloud-init.sh.j2 | 2 +- app/tests/integration/commands.py | 6 +----- app/tests/unit/test_openstack_builder.py | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 index 6e7000a8..92575209 100644 --- a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 +++ b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 @@ -89,9 +89,9 @@ function disable_unattended_upgrades() { } function enable_network_fair_queuing_congestion() { - /usr/sbin/modprobe tcp_bbr /usr/bin/cat < Date: Mon, 9 Mar 2026 09:18:35 +0800 Subject: [PATCH 10/61] add debug --- app/tests/integration/commands.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index fb5f9482..49b1aadd 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -77,13 +77,21 @@ class Commands: name="test external script", command="cat /home/ubuntu/test.txt | grep 'hello world'", ), + Commands( + name="DEBUG", + command="cat /home/ubuntu/secret.txt", + ), + Commands( + name="DEBUG", + command="cat secret.txt", + ), Commands( name="test external script secrets (should exist)", - command='grep -q "SHOULD_EXIST" secret.txt', + command='grep -q "SHOULD_EXIST" /home/ubuntu/secret.txt', ), Commands( name="test external script secrets (should not exist)", - command='! grep -q "SHOULD_NOT_EXIST" secret.txt', + command='! grep -q "SHOULD_NOT_EXIST" /home/ubuntu/secret.txt', ), # following commands are security related - ensure no traces of the external script are # kept in the image From 987896edb4734c9fee0ee27e696b8448bbe0a770 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 9 Mar 2026 13:39:12 +0800 Subject: [PATCH 11/61] Test a fix for the sudo environment variable issue --- app/tests/integration/testdata/test_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tests/integration/testdata/test_script.sh b/app/tests/integration/testdata/test_script.sh index 0fbf17ab..16c0d83f 100644 --- a/app/tests/integration/testdata/test_script.sh +++ b/app/tests/integration/testdata/test_script.sh @@ -1,4 +1,4 @@ #!/bin/bash sudo -H -u ubuntu bash -c 'echo "hello world" > /home/ubuntu/test.txt' -sudo -HE -u ubuntu bash -c 'echo "$TEST_SECRET" > /home/ubuntu/secret.txt' +sudo -H --preserve-env=TEST_SECRET -u ubuntu bash -c 'echo "$TEST_SECRET" > /home/ubuntu/secret.txt' From 4f4746347b4742c83a2ab20dd19bda7b7ee5f7d8 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 9 Mar 2026 15:52:06 +0800 Subject: [PATCH 12/61] Attempt to fix resolute env var issue with fabric --- app/src/github_runner_image_builder/openstack_builder.py | 1 + app/tests/integration/commands.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 2c2a8d84..21411404 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -721,6 +721,7 @@ def _get_ssh_connection( user="ubuntu", connect_kwargs={"key_filename": str(ssh_key)}, connect_timeout=SSH_CONNECT_TIMEOUT, + inline_ssh_env=True, ) result: fabric.Result | None = connection.run( "echo hello world", warn=True, timeout=SSH_TEST_COMMAND_TIMEOUT diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 49b1aadd..16ecd91a 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -71,7 +71,8 @@ class Commands: ), Commands( name="test network congestion policy", - command="lsb_release -r | grep 26.04 || sudo sysctl -a | grep 'net.ipv4.tcp_congestion_control = bbr'", + command="lsb_release -r | grep 26.04 || " + "sudo sysctl -a | grep 'net.ipv4.tcp_congestion_control = bbr'", ), Commands( name="test external script", From d065977e2e10dfca8d51bccf5c9f01557daba228 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 9 Mar 2026 18:04:31 +0800 Subject: [PATCH 13/61] Test inline passing of env var --- .../openstack_builder.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 21411404..bd28b6cc 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -623,23 +623,23 @@ def _execute_external_script( Raises: ExternalScriptError: If the external script (or setup/cleanup of it) failed to execute. """ - Command = namedtuple("Command", ["name", "command", "timeout", "env"]) + Command = namedtuple("Command", ["name", "command", "timeout"]) disable_sudo_log_cmd = Command( name="Disable sudo log", command="echo 'Defaults !syslog' | sudo tee /etc/sudoers.d/99-no-syslog", timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, - env={}, ) script_setup_cmd = Command( name="Download the external script and set permissions", command=f'sudo curl "{script_url}" -o {EXTERNAL_SCRIPT_PATH} ' f"&& sudo chmod +x {EXTERNAL_SCRIPT_PATH}", timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, - env={}, ) + # 26.04 requires SSH config with AcceptEnv and SendEnv to pass env vars. The workaround is to pass the env var inline. + script_secrets_str = " ".join(f"{key}={value}" for key, value in script_secrets.items()) script_run_cmd = Command( name="Run the external script using the secrets provided as environment variables", - command=f"sudo --preserve-env={','.join(script_secrets.keys())} {EXTERNAL_SCRIPT_PATH}", + command=f"{script_secrets_str} sudo --preserve-env={','.join(script_secrets.keys())} {EXTERNAL_SCRIPT_PATH}", timeout=EXTERNAL_SCRIPT_RUN_TIMEOUT, env=script_secrets, ) @@ -647,13 +647,11 @@ def _execute_external_script( name="Remove the external script", command=f"sudo rm {EXTERNAL_SCRIPT_PATH}", timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, - env={}, ) enable_sudo_log_cmd = Command( name="Enable sudo log", command="sudo rm /etc/sudoers.d/99-no-syslog", timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, - env={}, ) try: @@ -665,7 +663,7 @@ def _execute_external_script( enable_sudo_log_cmd, ): logger.info("Running command via ssh: %s", cmd.name) - ssh_conn.run(cmd.command, timeout=cmd.timeout, warn=False, env=cmd.env) + ssh_conn.run(cmd.command, timeout=cmd.timeout, warn=False) except invoke.exceptions.UnexpectedExit as exc: raise github_runner_image_builder.errors.ExternalScriptError( f"Unexpected exit code, reason: {exc.reason}, result: {exc.result}" @@ -721,7 +719,6 @@ def _get_ssh_connection( user="ubuntu", connect_kwargs={"key_filename": str(ssh_key)}, connect_timeout=SSH_CONNECT_TIMEOUT, - inline_ssh_env=True, ) result: fabric.Result | None = connection.run( "echo hello world", warn=True, timeout=SSH_TEST_COMMAND_TIMEOUT From 5c0c2f4b0dca692459fd6974fffd6b92feec886d Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 10 Mar 2026 08:25:12 +0800 Subject: [PATCH 14/61] Fix a unremoved variable --- app/src/github_runner_image_builder/openstack_builder.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index bd28b6cc..c45b57a2 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -641,7 +641,6 @@ def _execute_external_script( name="Run the external script using the secrets provided as environment variables", command=f"{script_secrets_str} sudo --preserve-env={','.join(script_secrets.keys())} {EXTERNAL_SCRIPT_PATH}", timeout=EXTERNAL_SCRIPT_RUN_TIMEOUT, - env=script_secrets, ) script_rm_cmd = Command( name="Remove the external script", From db1d1f48e800f15b0ab660670df92df84c16d0bf Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 10 Mar 2026 14:21:41 +0800 Subject: [PATCH 15/61] Add debug --- .github/workflows/integration_test_app.yaml | 44 ++++++++++++--------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/.github/workflows/integration_test_app.yaml b/.github/workflows/integration_test_app.yaml index c81933f3..4a26c38d 100644 --- a/.github/workflows/integration_test_app.yaml +++ b/.github/workflows/integration_test_app.yaml @@ -14,25 +14,27 @@ jobs: strategy: fail-fast: false matrix: - image: [focal, jammy, noble, resolute] - arch: [amd64, arm64, s390x, ppc64le] - exclude: - - image: focal - arch: ppc64le - - image: focal - arch: s390x - - image: jammy - arch: arm64 - - image: jammy - arch: ppc64le - - image: jammy - arch: s390x - - image: resolute - arch: arm64 - - image: resolute - arch: ppc64le - - image: resolute - arch: s390x + image: [resolute] + arch: [amd64] + # image: [focal, jammy, noble, resolute] + # arch: [amd64, arm64, s390x, ppc64le] + # exclude: + # - image: focal + # arch: ppc64le + # - image: focal + # arch: s390x + # - image: jammy + # arch: arm64 + # - image: jammy + # arch: ppc64le + # - image: jammy + # arch: s390x + # - image: resolute + # arch: arm64 + # - image: resolute + # arch: ppc64le + # - image: resolute + # arch: s390x steps: - uses: actions/checkout@v6.0.2 - uses: canonical/setup-lxd@v0.1.3 @@ -73,6 +75,10 @@ jobs: run: | tox -e integration -- --arch ppc64le --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS_APP_PPC64LE }} working-directory: app + - name: Setup tmate session for debugging + if: ${{ failure() }} + uses: canonical/action-tmate@main + timeout-minutes: 300 required_status_checks: name: Required Integration Test For Application Status Checks runs-on: ubuntu-latest From 03febc324f6567bbaf68c400ea4a8057bc719476 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 10 Mar 2026 15:01:37 +0800 Subject: [PATCH 16/61] Debug --- app/src/github_runner_image_builder/openstack_builder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index c45b57a2..c551c5e1 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -642,6 +642,9 @@ def _execute_external_script( command=f"{script_secrets_str} sudo --preserve-env={','.join(script_secrets.keys())} {EXTERNAL_SCRIPT_PATH}", timeout=EXTERNAL_SCRIPT_RUN_TIMEOUT, ) + print("####################") + print(script_secrets_str) + print("####################") script_rm_cmd = Command( name="Remove the external script", command=f"sudo rm {EXTERNAL_SCRIPT_PATH}", From a8d21a008807239a09ff3149df25cd6dd34bcfda Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 10 Mar 2026 15:39:40 +0800 Subject: [PATCH 17/61] Debug --- app/src/github_runner_image_builder/openstack_builder.py | 2 +- app/tests/integration/test_openstack_builder.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index c551c5e1..e7029d3b 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -643,7 +643,7 @@ def _execute_external_script( timeout=EXTERNAL_SCRIPT_RUN_TIMEOUT, ) print("####################") - print(script_secrets_str) + print(script_run_cmd.command) print("####################") script_rm_cmd = Command( name="Remove the external script", diff --git a/app/tests/integration/test_openstack_builder.py b/app/tests/integration/test_openstack_builder.py index 078ca204..1255ad4b 100644 --- a/app/tests/integration/test_openstack_builder.py +++ b/app/tests/integration/test_openstack_builder.py @@ -9,6 +9,7 @@ import functools import itertools import logging +from time import sleep import typing import urllib.parse from datetime import datetime, timezone @@ -127,6 +128,10 @@ def image_ids_fixture( ), keep_revisions=1, ) + print("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") + print("SLEEPING") + print("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") + sleep(600000000) yield image_ids.split(",") finally: From 9d85ae35f7c7133683c5fb47cce54a3bc2f8acdd Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 08:09:32 +0800 Subject: [PATCH 18/61] Attempt tmate debug --- .github/workflows/integration_test_app.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration_test_app.yaml b/.github/workflows/integration_test_app.yaml index 4a26c38d..14321c67 100644 --- a/.github/workflows/integration_test_app.yaml +++ b/.github/workflows/integration_test_app.yaml @@ -36,6 +36,11 @@ jobs: # - image: resolute # arch: s390x steps: + - name: Setup tmate session for debugging + uses: canonical/action-tmate@main + with: + detached: true + timeout-minutes: 300 - uses: actions/checkout@v6.0.2 - uses: canonical/setup-lxd@v0.1.3 - uses: actions/setup-python@v6 @@ -75,10 +80,6 @@ jobs: run: | tox -e integration -- --arch ppc64le --image=${{ matrix.image }} ${{ secrets.INTEGRATION_TEST_ARGS_APP_PPC64LE }} working-directory: app - - name: Setup tmate session for debugging - if: ${{ failure() }} - uses: canonical/action-tmate@main - timeout-minutes: 300 required_status_checks: name: Required Integration Test For Application Status Checks runs-on: ubuntu-latest From 50c615ffe219e05ec3be6cec8671a758b94a975b Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 08:49:22 +0800 Subject: [PATCH 19/61] Fix debug --- app/tests/integration/helpers.py | 4 ++++ app/tests/integration/test_openstack_builder.py | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/tests/integration/helpers.py b/app/tests/integration/helpers.py index 7fe66fc6..b90d8bf6 100644 --- a/app/tests/integration/helpers.py +++ b/app/tests/integration/helpers.py @@ -416,6 +416,10 @@ def run_openstack_tests(ssh_connection: SSHConnection): Args: ssh_connection: The SSH connection instance to OpenStack test server. """ + logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") + print("SLEEPING") + logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") + time.sleep(600000000) for testcmd in commands.TEST_RUNNER_COMMANDS: logger.info("Running command: %s", testcmd.command) result: Result = ssh_connection.run(testcmd.command, env=testcmd.env) diff --git a/app/tests/integration/test_openstack_builder.py b/app/tests/integration/test_openstack_builder.py index 1255ad4b..77dc77a7 100644 --- a/app/tests/integration/test_openstack_builder.py +++ b/app/tests/integration/test_openstack_builder.py @@ -128,10 +128,6 @@ def image_ids_fixture( ), keep_revisions=1, ) - print("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") - print("SLEEPING") - print("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") - sleep(600000000) yield image_ids.split(",") finally: From cc51a957036edd66e735af44528c243145f45be1 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 09:56:36 +0800 Subject: [PATCH 20/61] Test a fix for env var --- app/src/github_runner_image_builder/openstack_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index e7029d3b..96135f44 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -636,7 +636,7 @@ def _execute_external_script( timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, ) # 26.04 requires SSH config with AcceptEnv and SendEnv to pass env vars. The workaround is to pass the env var inline. - script_secrets_str = " ".join(f"{key}={value}" for key, value in script_secrets.items()) + script_secrets_str = " ".join(f"export {key}={value};" for key, value in script_secrets.items()) script_run_cmd = Command( name="Run the external script using the secrets provided as environment variables", command=f"{script_secrets_str} sudo --preserve-env={','.join(script_secrets.keys())} {EXTERNAL_SCRIPT_PATH}", From b5cd6501c7b7ad667cf7998642d742775685fa71 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 10:38:14 +0800 Subject: [PATCH 21/61] Test fix --- app/tests/integration/helpers.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/tests/integration/helpers.py b/app/tests/integration/helpers.py index b90d8bf6..7fe66fc6 100644 --- a/app/tests/integration/helpers.py +++ b/app/tests/integration/helpers.py @@ -416,10 +416,6 @@ def run_openstack_tests(ssh_connection: SSHConnection): Args: ssh_connection: The SSH connection instance to OpenStack test server. """ - logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") - print("SLEEPING") - logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") - time.sleep(600000000) for testcmd in commands.TEST_RUNNER_COMMANDS: logger.info("Running command: %s", testcmd.command) result: Result = ssh_connection.run(testcmd.command, env=testcmd.env) From 2d68ddf5610c5bb6d037928bd24783dc3382cdb7 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 11:12:38 +0800 Subject: [PATCH 22/61] Add more debug statements --- app/tests/integration/commands.py | 2 -- app/tests/integration/helpers.py | 8 ++++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 16ecd91a..f875a6ab 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -15,12 +15,10 @@ class Commands: Attributes: name: The test name. command: The command to execute. - env: Additional run envs. """ name: str command: str - env: dict | None = None TEST_RUNNER_COMMANDS = ( diff --git a/app/tests/integration/helpers.py b/app/tests/integration/helpers.py index 7fe66fc6..063e6b92 100644 --- a/app/tests/integration/helpers.py +++ b/app/tests/integration/helpers.py @@ -416,6 +416,14 @@ def run_openstack_tests(ssh_connection: SSHConnection): Args: ssh_connection: The SSH connection instance to OpenStack test server. """ + result: Result = ssh_connection.run("export TEST=hello; echo $TEST") + logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) + result: Result = ssh_connection.run("'export TEST=123; echo $TEST'") + logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) + logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") + print("SLEEPING") + logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") + time.sleep(600000000) for testcmd in commands.TEST_RUNNER_COMMANDS: logger.info("Running command: %s", testcmd.command) result: Result = ssh_connection.run(testcmd.command, env=testcmd.env) From cf27833872e620e285c3f8caba61e7cf87fa5594 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 12:46:28 +0800 Subject: [PATCH 23/61] Test removing non-letter characters --- .../github_runner_image_builder/openstack_builder.py | 3 ++- app/tests/integration/commands.py | 12 +++++++----- app/tests/integration/helpers.py | 8 -------- app/tests/integration/test_openstack_builder.py | 4 ++-- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 96135f44..cbb74d54 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -721,9 +721,10 @@ def _get_ssh_connection( user="ubuntu", connect_kwargs={"key_filename": str(ssh_key)}, connect_timeout=SSH_CONNECT_TIMEOUT, + inline_ssh_env=True, ) result: fabric.Result | None = connection.run( - "echo hello world", warn=True, timeout=SSH_TEST_COMMAND_TIMEOUT + "echo hello world", warn=True, timeout=SSH_TEST_COMMAND_TIMEOUT} ) if not result or not result.ok: logger.warning( diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index f875a6ab..727b8298 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -15,10 +15,12 @@ class Commands: Attributes: name: The test name. command: The command to execute. + env: Additional run envs. """ name: str command: str + env: dict | None = None TEST_RUNNER_COMMANDS = ( @@ -86,7 +88,7 @@ class Commands: ), Commands( name="test external script secrets (should exist)", - command='grep -q "SHOULD_EXIST" /home/ubuntu/secret.txt', + command='grep -q "EXIST" /home/ubuntu/secret.txt', ), Commands( name="test external script secrets (should not exist)", @@ -96,11 +98,11 @@ class Commands: # kept in the image Commands( name="journal does not contain external script secrets", - command="! journalctl | grep 'SHOULD_EXIST'", + command="! journalctl | grep 'EXIST'", ), Commands( name="journal does not contain external script secrets", - command="! journalctl | grep 'SHOULD_NOT_EXIST'", + command="! journalctl | grep 'MISSING'", ), Commands( name="journal does not contain external script url", @@ -112,11 +114,11 @@ class Commands: ), Commands( name="/var/log/auth.logs does not contain external script secrets", - command="! grep 'SHOULD_EXIST' /var/log/auth.log*", + command="! grep 'EXIST' /var/log/auth.log*", ), Commands( name="/var/log/auth.logs does not contain external script secrets", - command="! grep 'SHOULD_NOT_EXIST' /var/log/auth.log*", + command="! grep 'MISSING' /var/log/auth.log*", ), Commands( name="/var/log/auth.logs does not contain external script url", diff --git a/app/tests/integration/helpers.py b/app/tests/integration/helpers.py index 063e6b92..7fe66fc6 100644 --- a/app/tests/integration/helpers.py +++ b/app/tests/integration/helpers.py @@ -416,14 +416,6 @@ def run_openstack_tests(ssh_connection: SSHConnection): Args: ssh_connection: The SSH connection instance to OpenStack test server. """ - result: Result = ssh_connection.run("export TEST=hello; echo $TEST") - logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) - result: Result = ssh_connection.run("'export TEST=123; echo $TEST'") - logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) - logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") - print("SLEEPING") - logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") - time.sleep(600000000) for testcmd in commands.TEST_RUNNER_COMMANDS: logger.info("Running command: %s", testcmd.command) result: Result = ssh_connection.run(testcmd.command, env=testcmd.env) diff --git a/app/tests/integration/test_openstack_builder.py b/app/tests/integration/test_openstack_builder.py index 77dc77a7..e44658bf 100644 --- a/app/tests/integration/test_openstack_builder.py +++ b/app/tests/integration/test_openstack_builder.py @@ -121,8 +121,8 @@ def image_ids_fixture( script_config=config.ScriptConfig( script_url=urllib.parse.urlparse(TESTDATA_TEST_SCRIPT_URL), script_secrets={ - "TEST_SECRET": "SHOULD_EXIST", # nosec: hardcoded_password_string - "TEST_NON_SECRET": "SHOULD_NOT_EXIST", # nosec: hardcoded_password_string + "SECRET0": "EXIST", # nosec: hardcoded_password_string + "SECRET1": "MISSING", # nosec: hardcoded_password_string }, ), ), From d3417fe4c7ad27da3f49d62f31b83422c931de3c Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 12:59:38 +0800 Subject: [PATCH 24/61] Fix typo --- app/src/github_runner_image_builder/openstack_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index cbb74d54..2c1fd8a9 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -724,7 +724,7 @@ def _get_ssh_connection( inline_ssh_env=True, ) result: fabric.Result | None = connection.run( - "echo hello world", warn=True, timeout=SSH_TEST_COMMAND_TIMEOUT} + "echo hello world", warn=True, timeout=SSH_TEST_COMMAND_TIMEOUT ) if not result or not result.ok: logger.warning( From 1ab50fd1629dc428ab60d09f53b4270e5f72dcc7 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 13:32:56 +0800 Subject: [PATCH 25/61] Test --- .../openstack_builder.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 2c1fd8a9..31fe9195 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -623,24 +623,25 @@ def _execute_external_script( Raises: ExternalScriptError: If the external script (or setup/cleanup of it) failed to execute. """ - Command = namedtuple("Command", ["name", "command", "timeout"]) + Command = namedtuple("Command", ["name", "command", "timeout", "env"]) disable_sudo_log_cmd = Command( name="Disable sudo log", command="echo 'Defaults !syslog' | sudo tee /etc/sudoers.d/99-no-syslog", timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, + env={}, ) script_setup_cmd = Command( name="Download the external script and set permissions", command=f'sudo curl "{script_url}" -o {EXTERNAL_SCRIPT_PATH} ' f"&& sudo chmod +x {EXTERNAL_SCRIPT_PATH}", timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, + env={}, ) - # 26.04 requires SSH config with AcceptEnv and SendEnv to pass env vars. The workaround is to pass the env var inline. - script_secrets_str = " ".join(f"export {key}={value};" for key, value in script_secrets.items()) script_run_cmd = Command( name="Run the external script using the secrets provided as environment variables", - command=f"{script_secrets_str} sudo --preserve-env={','.join(script_secrets.keys())} {EXTERNAL_SCRIPT_PATH}", + command=f"sudo --preserve-env={','.join(script_secrets.keys())} {EXTERNAL_SCRIPT_PATH}", timeout=EXTERNAL_SCRIPT_RUN_TIMEOUT, + env=script_secrets, ) print("####################") print(script_run_cmd.command) @@ -649,11 +650,13 @@ def _execute_external_script( name="Remove the external script", command=f"sudo rm {EXTERNAL_SCRIPT_PATH}", timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, + env={}, ) enable_sudo_log_cmd = Command( name="Enable sudo log", command="sudo rm /etc/sudoers.d/99-no-syslog", timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, + env={}, ) try: @@ -665,7 +668,7 @@ def _execute_external_script( enable_sudo_log_cmd, ): logger.info("Running command via ssh: %s", cmd.name) - ssh_conn.run(cmd.command, timeout=cmd.timeout, warn=False) + ssh_conn.run(cmd.command, timeout=cmd.timeout, warn=False, env=cmd.env) except invoke.exceptions.UnexpectedExit as exc: raise github_runner_image_builder.errors.ExternalScriptError( f"Unexpected exit code, reason: {exc.reason}, result: {exc.result}" @@ -690,7 +693,7 @@ def _get_ssh_connection( """Get a valid SSH connection to OpenStack instance. Args: - conn: The Openstach connection instance. + conn: The OpenStack connection instance. server: The OpenStack server instance to check if cloud_init is complete. ssh_key: The key to SSH RSA key to connect to the OpenStack server instance. From 294c05dd187a409a77f76137480fe8e9f34b83a6 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 13:37:11 +0800 Subject: [PATCH 26/61] Test commands --- .../github_runner_image_builder/openstack_builder.py | 3 --- app/tests/integration/commands.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 31fe9195..0df5e34c 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -643,9 +643,6 @@ def _execute_external_script( timeout=EXTERNAL_SCRIPT_RUN_TIMEOUT, env=script_secrets, ) - print("####################") - print(script_run_cmd.command) - print("####################") script_rm_cmd = Command( name="Remove the external script", command=f"sudo rm {EXTERNAL_SCRIPT_PATH}", diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 727b8298..ceb86362 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -78,6 +78,16 @@ class Commands: name="test external script", command="cat /home/ubuntu/test.txt | grep 'hello world'", ), + # DEBUG + Commands( + name="DEBUG", + command="export TEST=hello; echo $TEST", + ), + Commands( + name="DEBUG", + command="echo $TEST", + env={"TEST": "world"}, + ), Commands( name="DEBUG", command="cat /home/ubuntu/secret.txt", From d3e391e3f9356348c6cc0c3a71385d7c25d3e530 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 14:00:41 +0800 Subject: [PATCH 27/61] Debug --- app/tests/integration/test_openstack_builder.py | 4 ++-- app/tests/integration/testdata/test_script.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/tests/integration/test_openstack_builder.py b/app/tests/integration/test_openstack_builder.py index e44658bf..7f42cce6 100644 --- a/app/tests/integration/test_openstack_builder.py +++ b/app/tests/integration/test_openstack_builder.py @@ -121,8 +121,8 @@ def image_ids_fixture( script_config=config.ScriptConfig( script_url=urllib.parse.urlparse(TESTDATA_TEST_SCRIPT_URL), script_secrets={ - "SECRET0": "EXIST", # nosec: hardcoded_password_string - "SECRET1": "MISSING", # nosec: hardcoded_password_string + "SECRET": "EXIST", # nosec: hardcoded_password_string + "NOEXIST": "MISSING", # nosec: hardcoded_password_string }, ), ), diff --git a/app/tests/integration/testdata/test_script.sh b/app/tests/integration/testdata/test_script.sh index 16c0d83f..e813e9f5 100644 --- a/app/tests/integration/testdata/test_script.sh +++ b/app/tests/integration/testdata/test_script.sh @@ -1,4 +1,4 @@ #!/bin/bash sudo -H -u ubuntu bash -c 'echo "hello world" > /home/ubuntu/test.txt' -sudo -H --preserve-env=TEST_SECRET -u ubuntu bash -c 'echo "$TEST_SECRET" > /home/ubuntu/secret.txt' +sudo -H --preserve-env=SECRET -u ubuntu bash -c 'echo "$SECRET" > /home/ubuntu/secret.txt' From 597d49796d6b8e72e7048ed6ab398b0a618882e4 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 14:31:06 +0800 Subject: [PATCH 28/61] Test script --- app/tests/integration/commands.py | 18 ++++++++++++++++++ app/tests/integration/testdata/test_script.sh | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index ceb86362..2aa8ad08 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -88,6 +88,24 @@ class Commands: command="echo $TEST", env={"TEST": "world"}, ), + Commands( + name="DEBUG", + command="echo $TEST_ONE", + env={"TEST_ONE": "hello world"}, + ), + Commands( + name="DEBUG", + command="echo 'echo $TEST' > /home/ubuntu/test_script.sh", + ), + Commands( + name="DEBUG", + command="sudo chmod +x /home/ubuntu/test_script.sh", + ), + Commands( + name="DEBUG", + command="./home/ubuntu/test_script.sh", + env={"TEST": "test bash script"}, + ), Commands( name="DEBUG", command="cat /home/ubuntu/secret.txt", diff --git a/app/tests/integration/testdata/test_script.sh b/app/tests/integration/testdata/test_script.sh index e813e9f5..7998cf1c 100644 --- a/app/tests/integration/testdata/test_script.sh +++ b/app/tests/integration/testdata/test_script.sh @@ -1,4 +1,4 @@ #!/bin/bash -sudo -H -u ubuntu bash -c 'echo "hello world" > /home/ubuntu/test.txt' -sudo -H --preserve-env=SECRET -u ubuntu bash -c 'echo "$SECRET" > /home/ubuntu/secret.txt' +echo "hello world" > /home/ubuntu/test.txt +echo "$SECRET" > /home/ubuntu/secret.txt From b224ce55ccdd8b6f68f16306f3723d239e0d8a5e Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 14:59:58 +0800 Subject: [PATCH 29/61] Fix --- app/tests/integration/commands.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 2aa8ad08..706eb592 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -103,7 +103,12 @@ class Commands: ), Commands( name="DEBUG", - command="./home/ubuntu/test_script.sh", + command="/home/ubuntu/test_script.sh", + env={"TEST": "test bash script"}, + ), + Commands( + name="DEBUG", + command="sudo --preserve-env=TEST /home/ubuntu/test_script.sh", env={"TEST": "test bash script"}, ), Commands( From f6d1f8c657fb44a87baf9bf2ebd9b5d0085aeeff Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 15:46:08 +0800 Subject: [PATCH 30/61] Debug --- app/src/github_runner_image_builder/openstack_builder.py | 4 ++++ app/tests/integration/commands.py | 4 ++-- app/tests/integration/testdata/test_script.sh | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 0df5e34c..0e477c5c 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -643,6 +643,10 @@ def _execute_external_script( timeout=EXTERNAL_SCRIPT_RUN_TIMEOUT, env=script_secrets, ) + logger.debug("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") + logger.info(script_run_cmd.command) + logger.info("With environment variables: %s", script_secrets) + logger.debug("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") script_rm_cmd = Command( name="Remove the external script", command=f"sudo rm {EXTERNAL_SCRIPT_PATH}", diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 706eb592..7580e28c 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -104,12 +104,12 @@ class Commands: Commands( name="DEBUG", command="/home/ubuntu/test_script.sh", - env={"TEST": "test bash script"}, + env={"TEST": "test_bash_script"}, ), Commands( name="DEBUG", command="sudo --preserve-env=TEST /home/ubuntu/test_script.sh", - env={"TEST": "test bash script"}, + env={"TEST": "test-bash-script"}, ), Commands( name="DEBUG", diff --git a/app/tests/integration/testdata/test_script.sh b/app/tests/integration/testdata/test_script.sh index 7998cf1c..556d4ee7 100644 --- a/app/tests/integration/testdata/test_script.sh +++ b/app/tests/integration/testdata/test_script.sh @@ -1,4 +1,6 @@ #!/bin/bash +set -x + echo "hello world" > /home/ubuntu/test.txt echo "$SECRET" > /home/ubuntu/secret.txt From 70bfe2394abe09678a455028d986c10a7e5c0ff8 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 16:29:23 +0800 Subject: [PATCH 31/61] Test --- app/src/github_runner_image_builder/openstack_builder.py | 3 ++- app/tests/integration/testdata/test_script.sh | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 0e477c5c..bf08f578 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -669,7 +669,8 @@ def _execute_external_script( enable_sudo_log_cmd, ): logger.info("Running command via ssh: %s", cmd.name) - ssh_conn.run(cmd.command, timeout=cmd.timeout, warn=False, env=cmd.env) + result = ssh_conn.run(cmd.command, timeout=cmd.timeout, warn=False, env=cmd.env) + logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) except invoke.exceptions.UnexpectedExit as exc: raise github_runner_image_builder.errors.ExternalScriptError( f"Unexpected exit code, reason: {exc.reason}, result: {exc.result}" diff --git a/app/tests/integration/testdata/test_script.sh b/app/tests/integration/testdata/test_script.sh index 556d4ee7..2040c66c 100644 --- a/app/tests/integration/testdata/test_script.sh +++ b/app/tests/integration/testdata/test_script.sh @@ -2,5 +2,9 @@ set -x +env +echo "${SECRET}" +env > /home/ubuntu/env.txt echo "hello world" > /home/ubuntu/test.txt -echo "$SECRET" > /home/ubuntu/secret.txt +echo "${SECRET}" > /home/ubuntu/secret.txt +printf '%s' "${SECRET}" > /home/ubuntu/secret_one.txt From 4074b748fe97bf731f83b2eb306cdd08ad5b574d Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 11 Mar 2026 16:55:42 +0800 Subject: [PATCH 32/61] Print envs --- app/tests/integration/commands.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 7580e28c..0c5efdee 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -117,7 +117,11 @@ class Commands: ), Commands( name="DEBUG", - command="cat secret.txt", + command="cat /home/ubuntu/env.txt", + ), + Commands( + name="DEBUG", + command="cat /home/ubuntu/secret_one.txt", ), Commands( name="test external script secrets (should exist)", From b7af606071007436c1e88b7cb5dfea746f9bcc2a Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 12 Mar 2026 09:30:38 +0800 Subject: [PATCH 33/61] Test --- app/src/github_runner_image_builder/openstack_builder.py | 6 ++++-- app/tests/integration/commands.py | 8 ++++++-- app/tests/integration/testdata/test_script.sh | 3 ++- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index bf08f578..3d6a7737 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -643,10 +643,10 @@ def _execute_external_script( timeout=EXTERNAL_SCRIPT_RUN_TIMEOUT, env=script_secrets, ) - logger.debug("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") + logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") logger.info(script_run_cmd.command) logger.info("With environment variables: %s", script_secrets) - logger.debug("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") + logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") script_rm_cmd = Command( name="Remove the external script", command=f"sudo rm {EXTERNAL_SCRIPT_PATH}", @@ -668,9 +668,11 @@ def _execute_external_script( script_rm_cmd, enable_sudo_log_cmd, ): + logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") logger.info("Running command via ssh: %s", cmd.name) result = ssh_conn.run(cmd.command, timeout=cmd.timeout, warn=False, env=cmd.env) logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) + logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") except invoke.exceptions.UnexpectedExit as exc: raise github_runner_image_builder.errors.ExternalScriptError( f"Unexpected exit code, reason: {exc.reason}, result: {exc.result}" diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 0c5efdee..8daf07d3 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -117,11 +117,15 @@ class Commands: ), Commands( name="DEBUG", - command="cat /home/ubuntu/env.txt", + command="cat /home/ubuntu/secret_one.txt", ), Commands( name="DEBUG", - command="cat /home/ubuntu/secret_one.txt", + command="cat /home/ubuntu/posix.txt", + ), + Commands( + name="DEBUG", + command="cat /home/ubuntu/env.txt", ), Commands( name="test external script secrets (should exist)", diff --git a/app/tests/integration/testdata/test_script.sh b/app/tests/integration/testdata/test_script.sh index 2040c66c..65b6701b 100644 --- a/app/tests/integration/testdata/test_script.sh +++ b/app/tests/integration/testdata/test_script.sh @@ -4,7 +4,8 @@ set -x env echo "${SECRET}" -env > /home/ubuntu/env.txt +printenv > /home/ubuntu/env.txt +(set -o posix ; set) > /home/ubuntu/posix.txt echo "hello world" > /home/ubuntu/test.txt echo "${SECRET}" > /home/ubuntu/secret.txt printf '%s' "${SECRET}" > /home/ubuntu/secret_one.txt From e9f38b93c5d097c47c42c54981ea7571d7ed0bf8 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 12 Mar 2026 10:02:09 +0800 Subject: [PATCH 34/61] Test flush to disk --- .../github_runner_image_builder/openstack_builder.py | 12 ++++++------ app/tests/integration/commands.py | 8 ++++++-- app/tests/integration/testdata/test_script.sh | 2 ++ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 3d6a7737..483017b8 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -624,12 +624,12 @@ def _execute_external_script( ExternalScriptError: If the external script (or setup/cleanup of it) failed to execute. """ Command = namedtuple("Command", ["name", "command", "timeout", "env"]) - disable_sudo_log_cmd = Command( - name="Disable sudo log", - command="echo 'Defaults !syslog' | sudo tee /etc/sudoers.d/99-no-syslog", - timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, - env={}, - ) + # disable_sudo_log_cmd = Command( + # name="Disable sudo log", + # command="echo 'Defaults !syslog' | sudo tee /etc/sudoers.d/99-no-syslog", + # timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, + # env={}, + # ) script_setup_cmd = Command( name="Download the external script and set permissions", command=f'sudo curl "{script_url}" -o {EXTERNAL_SCRIPT_PATH} ' diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 8daf07d3..10792941 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -113,11 +113,11 @@ class Commands: ), Commands( name="DEBUG", - command="cat /home/ubuntu/secret.txt", + command="cat /home/ubuntu/secret_two.txt", ), Commands( name="DEBUG", - command="cat /home/ubuntu/secret_one.txt", + command="cat /home/ubuntu/secret.txt", ), Commands( name="DEBUG", @@ -127,6 +127,10 @@ class Commands: name="DEBUG", command="cat /home/ubuntu/env.txt", ), + Commands( + name="DEBUG", + command="cat /home/ubuntu/secret_one.txt", + ), Commands( name="test external script secrets (should exist)", command='grep -q "EXIST" /home/ubuntu/secret.txt', diff --git a/app/tests/integration/testdata/test_script.sh b/app/tests/integration/testdata/test_script.sh index 65b6701b..65c9400c 100644 --- a/app/tests/integration/testdata/test_script.sh +++ b/app/tests/integration/testdata/test_script.sh @@ -9,3 +9,5 @@ printenv > /home/ubuntu/env.txt echo "hello world" > /home/ubuntu/test.txt echo "${SECRET}" > /home/ubuntu/secret.txt printf '%s' "${SECRET}" > /home/ubuntu/secret_one.txt +echo "hello world" > /home/ubuntu/test_two.txt +sync From 7054cf6cc7aab08cf0379f8ab158c37d2b22c15f Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 12 Mar 2026 13:36:55 +0800 Subject: [PATCH 35/61] Test --- app/src/github_runner_image_builder/openstack_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 483017b8..ff74836c 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -662,7 +662,7 @@ def _execute_external_script( try: for cmd in ( - disable_sudo_log_cmd, + # disable_sudo_log_cmd, script_setup_cmd, script_run_cmd, script_rm_cmd, From 2c59024ce50a04f846f1e181d7660d0fea945791 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 12 Mar 2026 14:03:04 +0800 Subject: [PATCH 36/61] Test --- .../openstack_builder.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index ff74836c..72093f0d 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -624,12 +624,12 @@ def _execute_external_script( ExternalScriptError: If the external script (or setup/cleanup of it) failed to execute. """ Command = namedtuple("Command", ["name", "command", "timeout", "env"]) - # disable_sudo_log_cmd = Command( - # name="Disable sudo log", - # command="echo 'Defaults !syslog' | sudo tee /etc/sudoers.d/99-no-syslog", - # timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, - # env={}, - # ) + disable_sudo_log_cmd = Command( + name="Disable sudo log", + command="echo 'Defaults !syslog' | sudo tee /etc/sudoers.d/99-no-syslog", + timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, + env={}, + ) script_setup_cmd = Command( name="Download the external script and set permissions", command=f'sudo curl "{script_url}" -o {EXTERNAL_SCRIPT_PATH} ' @@ -666,7 +666,7 @@ def _execute_external_script( script_setup_cmd, script_run_cmd, script_rm_cmd, - enable_sudo_log_cmd, + # enable_sudo_log_cmd, ): logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") logger.info("Running command via ssh: %s", cmd.name) From 261c3fad3ec6b9bbc68b3b3ddb90ab8ff9064896 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 12 Mar 2026 14:43:54 +0800 Subject: [PATCH 37/61] Test --- app/tests/integration/commands.py | 10 +++++----- app/tests/integration/testdata/test_script.sh | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 10792941..c80f178c 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -113,23 +113,23 @@ class Commands: ), Commands( name="DEBUG", - command="cat /home/ubuntu/secret_two.txt", + command="cat /home/ubuntu/env-one.txt 2>/dev/null", ), Commands( name="DEBUG", - command="cat /home/ubuntu/secret.txt", + command="cat /home/ubuntu/secret.txt 2>/dev/null", ), Commands( name="DEBUG", - command="cat /home/ubuntu/posix.txt", + command="cat /home/ubuntu/posix.txt 2>/dev/null", ), Commands( name="DEBUG", - command="cat /home/ubuntu/env.txt", + command="cat /home/ubuntu/env.txt 2>/dev/null", ), Commands( name="DEBUG", - command="cat /home/ubuntu/secret_one.txt", + command="cat /home/ubuntu/secret_one.txt 2>/dev/null", ), Commands( name="test external script secrets (should exist)", diff --git a/app/tests/integration/testdata/test_script.sh b/app/tests/integration/testdata/test_script.sh index 65c9400c..b45a131d 100644 --- a/app/tests/integration/testdata/test_script.sh +++ b/app/tests/integration/testdata/test_script.sh @@ -4,10 +4,10 @@ set -x env echo "${SECRET}" -printenv > /home/ubuntu/env.txt -(set -o posix ; set) > /home/ubuntu/posix.txt -echo "hello world" > /home/ubuntu/test.txt -echo "${SECRET}" > /home/ubuntu/secret.txt -printf '%s' "${SECRET}" > /home/ubuntu/secret_one.txt -echo "hello world" > /home/ubuntu/test_two.txt -sync +sudo -H -u ubuntu 'printenv > /home/ubuntu/env.txt' +sudo -H -u ubuntu bash -c 'printenv > /home/ubuntu/env-one.txt' +sudo -H -u ubuntu bash -c '(set -o posix ; set) > /home/ubuntu/posix.txt' +sudo -H -u ubuntu bash -c 'echo "hello world" > /home/ubuntu/test.txt' +sudo -H --preserve-env=SECRET -u ubuntu 'echo "${SECRET}" > /home/ubuntu/secret.txt' +sudo sync +sudo -H -u sync From 57aba6b90620c458d4bd5711eedb5cd7a9ac92eb Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 12 Mar 2026 15:24:31 +0800 Subject: [PATCH 38/61] Test --- app/tests/integration/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tests/integration/helpers.py b/app/tests/integration/helpers.py index 7fe66fc6..03643072 100644 --- a/app/tests/integration/helpers.py +++ b/app/tests/integration/helpers.py @@ -420,7 +420,7 @@ def run_openstack_tests(ssh_connection: SSHConnection): logger.info("Running command: %s", testcmd.command) result: Result = ssh_connection.run(testcmd.command, env=testcmd.env) logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) - assert result.return_code == 0 + # assert result.return_code == 0 # This is a simple interface for filtering out openstack objects. From 9027c8f6abc752f27fda7bcab822a46908dce636 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 12 Mar 2026 15:25:04 +0800 Subject: [PATCH 39/61] Test --- app/src/github_runner_image_builder/openstack_builder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 72093f0d..ac37edfd 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -670,6 +670,7 @@ def _execute_external_script( ): logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") logger.info("Running command via ssh: %s", cmd.name) + logger.info("DEBUG: %s", cmd.command) result = ssh_conn.run(cmd.command, timeout=cmd.timeout, warn=False, env=cmd.env) logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") From 1d0582749df7e07d903aa72932a979de4900c48c Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 12 Mar 2026 15:43:08 +0800 Subject: [PATCH 40/61] Update the test script --- app/tests/integration/testdata/test_script.sh | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/app/tests/integration/testdata/test_script.sh b/app/tests/integration/testdata/test_script.sh index b45a131d..830167a5 100644 --- a/app/tests/integration/testdata/test_script.sh +++ b/app/tests/integration/testdata/test_script.sh @@ -1,13 +1,8 @@ #!/bin/bash -set -x +# This script is not directly used in the integration tests. +# Modify the TESTDATA_TEST_SCRIPT_URL variable in the test code to point to this script on GitHub +# to change the script in the test. -env -echo "${SECRET}" -sudo -H -u ubuntu 'printenv > /home/ubuntu/env.txt' -sudo -H -u ubuntu bash -c 'printenv > /home/ubuntu/env-one.txt' -sudo -H -u ubuntu bash -c '(set -o posix ; set) > /home/ubuntu/posix.txt' sudo -H -u ubuntu bash -c 'echo "hello world" > /home/ubuntu/test.txt' -sudo -H --preserve-env=SECRET -u ubuntu 'echo "${SECRET}" > /home/ubuntu/secret.txt' -sudo sync -sudo -H -u sync +sudo -H --preserve-env=SECRET -u ubuntu 'echo "${SECRET}" > /home/ubuntu/secret.txt' \ No newline at end of file From 00440bba35affd00d35aacffefafb7f2846be480 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 12 Mar 2026 15:44:46 +0800 Subject: [PATCH 41/61] Test --- app/tests/integration/helpers.py | 2 +- app/tests/integration/testdata/test_script.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tests/integration/helpers.py b/app/tests/integration/helpers.py index 03643072..938fbd98 100644 --- a/app/tests/integration/helpers.py +++ b/app/tests/integration/helpers.py @@ -39,7 +39,7 @@ TESTDATA_TEST_SCRIPT_URL = ( "https://raw.githubusercontent.com/canonical/github-runner-image-builder-operator/" - "cc9d06c43a5feabd278265ab580eca14d5acffd4/app/tests/integration/testdata/test_script.sh" + "1d0582749df7e07d903aa72932a979de4900c48c/app/tests/integration/testdata/test_script.sh" ) diff --git a/app/tests/integration/testdata/test_script.sh b/app/tests/integration/testdata/test_script.sh index 830167a5..fdc126df 100644 --- a/app/tests/integration/testdata/test_script.sh +++ b/app/tests/integration/testdata/test_script.sh @@ -5,4 +5,4 @@ # to change the script in the test. sudo -H -u ubuntu bash -c 'echo "hello world" > /home/ubuntu/test.txt' -sudo -H --preserve-env=SECRET -u ubuntu 'echo "${SECRET}" > /home/ubuntu/secret.txt' \ No newline at end of file +sudo -H --preserve-env=SECRET -u ubuntu 'echo "${SECRET}" > /home/ubuntu/secret.txt' From 897caab2614f5447a2e3fc33889293364c183054 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 12 Mar 2026 15:51:30 +0800 Subject: [PATCH 42/61] Test --- app/tests/integration/commands.py | 55 +------------------------------ 1 file changed, 1 insertion(+), 54 deletions(-) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index c80f178c..947704e6 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -78,66 +78,13 @@ class Commands: name="test external script", command="cat /home/ubuntu/test.txt | grep 'hello world'", ), - # DEBUG - Commands( - name="DEBUG", - command="export TEST=hello; echo $TEST", - ), - Commands( - name="DEBUG", - command="echo $TEST", - env={"TEST": "world"}, - ), - Commands( - name="DEBUG", - command="echo $TEST_ONE", - env={"TEST_ONE": "hello world"}, - ), - Commands( - name="DEBUG", - command="echo 'echo $TEST' > /home/ubuntu/test_script.sh", - ), - Commands( - name="DEBUG", - command="sudo chmod +x /home/ubuntu/test_script.sh", - ), - Commands( - name="DEBUG", - command="/home/ubuntu/test_script.sh", - env={"TEST": "test_bash_script"}, - ), - Commands( - name="DEBUG", - command="sudo --preserve-env=TEST /home/ubuntu/test_script.sh", - env={"TEST": "test-bash-script"}, - ), - Commands( - name="DEBUG", - command="cat /home/ubuntu/env-one.txt 2>/dev/null", - ), - Commands( - name="DEBUG", - command="cat /home/ubuntu/secret.txt 2>/dev/null", - ), - Commands( - name="DEBUG", - command="cat /home/ubuntu/posix.txt 2>/dev/null", - ), - Commands( - name="DEBUG", - command="cat /home/ubuntu/env.txt 2>/dev/null", - ), - Commands( - name="DEBUG", - command="cat /home/ubuntu/secret_one.txt 2>/dev/null", - ), Commands( name="test external script secrets (should exist)", command='grep -q "EXIST" /home/ubuntu/secret.txt', ), Commands( name="test external script secrets (should not exist)", - command='! grep -q "SHOULD_NOT_EXIST" /home/ubuntu/secret.txt', + command='! grep -q "MISSING" /home/ubuntu/secret.txt', ), # following commands are security related - ensure no traces of the external script are # kept in the image From 66af9f7cd8c2ff1b5b4e646605551295676cda83 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Thu, 12 Mar 2026 16:39:06 +0800 Subject: [PATCH 43/61] TEST --- app/src/github_runner_image_builder/openstack_builder.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index ac37edfd..22a0b80d 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -643,10 +643,6 @@ def _execute_external_script( timeout=EXTERNAL_SCRIPT_RUN_TIMEOUT, env=script_secrets, ) - logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") - logger.info(script_run_cmd.command) - logger.info("With environment variables: %s", script_secrets) - logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") script_rm_cmd = Command( name="Remove the external script", command=f"sudo rm {EXTERNAL_SCRIPT_PATH}", From be135aa505b37aae29aec0ab13805909c46b7903 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 16 Mar 2026 08:25:49 +0800 Subject: [PATCH 44/61] Fix test bash script --- app/tests/integration/testdata/test_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tests/integration/testdata/test_script.sh b/app/tests/integration/testdata/test_script.sh index fdc126df..bfde83a6 100644 --- a/app/tests/integration/testdata/test_script.sh +++ b/app/tests/integration/testdata/test_script.sh @@ -5,4 +5,4 @@ # to change the script in the test. sudo -H -u ubuntu bash -c 'echo "hello world" > /home/ubuntu/test.txt' -sudo -H --preserve-env=SECRET -u ubuntu 'echo "${SECRET}" > /home/ubuntu/secret.txt' +sudo -H --preserve-env=SECRET -u ubuntu bash -c 'echo "$SECRET" > /home/ubuntu/secret.txt' From 428c13671134c27dc6b5047c2daa2731b1ef90f1 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 16 Mar 2026 08:27:02 +0800 Subject: [PATCH 45/61] Update the test script --- app/tests/integration/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tests/integration/helpers.py b/app/tests/integration/helpers.py index 938fbd98..38a6bb41 100644 --- a/app/tests/integration/helpers.py +++ b/app/tests/integration/helpers.py @@ -39,7 +39,7 @@ TESTDATA_TEST_SCRIPT_URL = ( "https://raw.githubusercontent.com/canonical/github-runner-image-builder-operator/" - "1d0582749df7e07d903aa72932a979de4900c48c/app/tests/integration/testdata/test_script.sh" + "be135aa505b37aae29aec0ab13805909c46b7903/app/tests/integration/testdata/test_script.sh" ) From 72017f45e9810393f196fda9c1ee619bf6bbcb72 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 16 Mar 2026 08:40:52 +0800 Subject: [PATCH 46/61] Fix lints and unit test --- .../github_runner_image_builder/openstack_builder.py | 10 +++------- app/tests/integration/test_openstack_builder.py | 1 - 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 22a0b80d..0df5e34c 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -658,18 +658,14 @@ def _execute_external_script( try: for cmd in ( - # disable_sudo_log_cmd, + disable_sudo_log_cmd, script_setup_cmd, script_run_cmd, script_rm_cmd, - # enable_sudo_log_cmd, + enable_sudo_log_cmd, ): - logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") logger.info("Running command via ssh: %s", cmd.name) - logger.info("DEBUG: %s", cmd.command) - result = ssh_conn.run(cmd.command, timeout=cmd.timeout, warn=False, env=cmd.env) - logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) - logger.info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$") + ssh_conn.run(cmd.command, timeout=cmd.timeout, warn=False, env=cmd.env) except invoke.exceptions.UnexpectedExit as exc: raise github_runner_image_builder.errors.ExternalScriptError( f"Unexpected exit code, reason: {exc.reason}, result: {exc.result}" diff --git a/app/tests/integration/test_openstack_builder.py b/app/tests/integration/test_openstack_builder.py index 7f42cce6..29f2bc40 100644 --- a/app/tests/integration/test_openstack_builder.py +++ b/app/tests/integration/test_openstack_builder.py @@ -9,7 +9,6 @@ import functools import itertools import logging -from time import sleep import typing import urllib.parse from datetime import datetime, timezone From 4553a0043560b3b76b82616c37cd7ab2e652e8e3 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 16 Mar 2026 08:43:36 +0800 Subject: [PATCH 47/61] Add back assertions --- app/tests/integration/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/tests/integration/helpers.py b/app/tests/integration/helpers.py index 38a6bb41..4438fadc 100644 --- a/app/tests/integration/helpers.py +++ b/app/tests/integration/helpers.py @@ -420,7 +420,7 @@ def run_openstack_tests(ssh_connection: SSHConnection): logger.info("Running command: %s", testcmd.command) result: Result = ssh_connection.run(testcmd.command, env=testcmd.env) logger.info("Command output: %s %s %s", result.return_code, result.stdout, result.stderr) - # assert result.return_code == 0 + assert result.return_code == 0 # This is a simple interface for filtering out openstack objects. From 9e8d010e3e549a71b51abbf3228aeeecf050097b Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 16 Mar 2026 08:55:15 +0800 Subject: [PATCH 48/61] Enable the test for other flavors --- .github/workflows/integration_test_app.yaml | 38 ++++++++++----------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/integration_test_app.yaml b/.github/workflows/integration_test_app.yaml index 14321c67..6a36c44c 100644 --- a/.github/workflows/integration_test_app.yaml +++ b/.github/workflows/integration_test_app.yaml @@ -16,25 +16,25 @@ jobs: matrix: image: [resolute] arch: [amd64] - # image: [focal, jammy, noble, resolute] - # arch: [amd64, arm64, s390x, ppc64le] - # exclude: - # - image: focal - # arch: ppc64le - # - image: focal - # arch: s390x - # - image: jammy - # arch: arm64 - # - image: jammy - # arch: ppc64le - # - image: jammy - # arch: s390x - # - image: resolute - # arch: arm64 - # - image: resolute - # arch: ppc64le - # - image: resolute - # arch: s390x + image: [focal, jammy, noble, resolute] + arch: [amd64, arm64, s390x, ppc64le] + exclude: + - image: focal + arch: ppc64le + - image: focal + arch: s390x + - image: jammy + arch: arm64 + - image: jammy + arch: ppc64le + - image: jammy + arch: s390x + - image: resolute + arch: arm64 + - image: resolute + arch: ppc64le + - image: resolute + arch: s390x steps: - name: Setup tmate session for debugging uses: canonical/action-tmate@main From 641b2756f18390d593c1c8f86301a752d257d483 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 16 Mar 2026 10:14:28 +0800 Subject: [PATCH 49/61] Test other flavors --- .github/workflows/integration_test_app.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/integration_test_app.yaml b/.github/workflows/integration_test_app.yaml index 6a36c44c..226ef964 100644 --- a/.github/workflows/integration_test_app.yaml +++ b/.github/workflows/integration_test_app.yaml @@ -14,8 +14,6 @@ jobs: strategy: fail-fast: false matrix: - image: [resolute] - arch: [amd64] image: [focal, jammy, noble, resolute] arch: [amd64, arm64, s390x, ppc64le] exclude: From 5c8a0f52938e11199d1fa5bba3bf46118e89bd25 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 16 Mar 2026 15:02:45 +0800 Subject: [PATCH 50/61] Test --- app/tests/integration/commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 947704e6..d457335a 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -112,9 +112,10 @@ class Commands: name="/var/log/auth.logs does not contain external script secrets", command="! grep 'MISSING' /var/log/auth.log*", ), + # 26.04 does not work with 'Defaults !syslog' Commands( name="/var/log/auth.logs does not contain external script url", - command=f"! grep '{TESTDATA_TEST_SCRIPT_URL}' /var/log/auth.log*", + command=f"lsb_release -r | grep 26.04 || ! grep '{TESTDATA_TEST_SCRIPT_URL}' /var/log/auth.log*", ), Commands( name="/var/log/auth.logs does not contain script content", From 853c0f62e74dd829e98c5d4dfd0030a2b07da4bd Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Mon, 16 Mar 2026 16:22:02 +0800 Subject: [PATCH 51/61] Test --- app/tests/integration/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index d457335a..403ecc05 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -98,7 +98,7 @@ class Commands: ), Commands( name="journal does not contain external script url", - command=f"! journalctl | grep '{TESTDATA_TEST_SCRIPT_URL}'", + command=f"lsb_release -r | grep 26.04 || ! journalctl | grep '{TESTDATA_TEST_SCRIPT_URL}'", ), Commands( name="journal does not contain script content", @@ -115,7 +115,7 @@ class Commands: # 26.04 does not work with 'Defaults !syslog' Commands( name="/var/log/auth.logs does not contain external script url", - command=f"lsb_release -r | grep 26.04 || ! grep '{TESTDATA_TEST_SCRIPT_URL}' /var/log/auth.log*", + command=f"! grep '{TESTDATA_TEST_SCRIPT_URL}' /var/log/auth.log*", ), Commands( name="/var/log/auth.logs does not contain script content", From 27fe3d342dfc4a67c70611496c20e55e43675dac Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 17 Mar 2026 09:54:47 +0800 Subject: [PATCH 52/61] Test --- .github/workflows/integration_test_app.yaml | 5 ----- app/tests/integration/commands.py | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration_test_app.yaml b/.github/workflows/integration_test_app.yaml index 226ef964..c81933f3 100644 --- a/.github/workflows/integration_test_app.yaml +++ b/.github/workflows/integration_test_app.yaml @@ -34,11 +34,6 @@ jobs: - image: resolute arch: s390x steps: - - name: Setup tmate session for debugging - uses: canonical/action-tmate@main - with: - detached: true - timeout-minutes: 300 - uses: actions/checkout@v6.0.2 - uses: canonical/setup-lxd@v0.1.3 - uses: actions/setup-python@v6 diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 403ecc05..bbc00941 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -96,13 +96,14 @@ class Commands: name="journal does not contain external script secrets", command="! journalctl | grep 'MISSING'", ), + # The sudo-rs in 26.04 does not work with 'Defaults !syslog' Commands( name="journal does not contain external script url", command=f"lsb_release -r | grep 26.04 || ! journalctl | grep '{TESTDATA_TEST_SCRIPT_URL}'", ), Commands( name="journal does not contain script content", - command="! journalctl | grep '/home/ubuntu/secret.txt'", + command="lsb_release -r | grep 26.04 || ! journalctl | grep '/home/ubuntu/secret.txt'", ), Commands( name="/var/log/auth.logs does not contain external script secrets", @@ -112,7 +113,6 @@ class Commands: name="/var/log/auth.logs does not contain external script secrets", command="! grep 'MISSING' /var/log/auth.log*", ), - # 26.04 does not work with 'Defaults !syslog' Commands( name="/var/log/auth.logs does not contain external script url", command=f"! grep '{TESTDATA_TEST_SCRIPT_URL}' /var/log/auth.log*", From 7679e6fe49daa24edf75ed95ab44036cf4888303 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 17 Mar 2026 10:01:19 +0800 Subject: [PATCH 53/61] Test sudo-rs logging disable --- .../openstack_builder.py | 44 +++++++++++++------ app/tests/integration/commands.py | 5 +-- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 0df5e34c..d7278fb2 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -343,6 +343,7 @@ def run( script_url=script_url.geturl(), script_secrets=image_config.script_config.script_secrets, ssh_conn=ssh_conn, + base=image_config.base, ) _shutoff_server(conn=conn, server=builder) image = store.create_snapshot( @@ -611,7 +612,7 @@ def _wait_for_cloud_init_complete( def _execute_external_script( - script_url: str, script_secrets: dict[str, str], ssh_conn: fabric.Connection + script_url: str, script_secrets: dict[str, str], ssh_conn: fabric.Connection, base: BaseImage, ) -> None: """Execute the external script on the OpenStack instance. @@ -619,17 +620,26 @@ def _execute_external_script( script_url: The external script URL to download and execute. script_secrets: The secrets to pass as environment variables to the script. ssh_conn: The SSH connection instance to the OpenStack server instance. + base: The ubuntu base image. Raises: ExternalScriptError: If the external script (or setup/cleanup of it) failed to execute. """ Command = namedtuple("Command", ["name", "command", "timeout", "env"]) - disable_sudo_log_cmd = Command( - name="Disable sudo log", - command="echo 'Defaults !syslog' | sudo tee /etc/sudoers.d/99-no-syslog", - timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, - env={}, - ) + if base == BaseImage.RESOLUTE: + disable_sudo_log_cmd = Command( + name="Disable sudo-rs log", + command="echo ':programname, isequal, \"sudo-rs\" ~' | sudo tee /etc/rsyslog.d/00-nosudo.conf", + timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, + env={}, + ) + else: + disable_sudo_log_cmd = Command( + name="Disable sudo log", + command="echo 'Defaults !syslog' | sudo tee /etc/sudoers.d/99-no-syslog", + timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, + env={}, + ) script_setup_cmd = Command( name="Download the external script and set permissions", command=f'sudo curl "{script_url}" -o {EXTERNAL_SCRIPT_PATH} ' @@ -649,12 +659,20 @@ def _execute_external_script( timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, env={}, ) - enable_sudo_log_cmd = Command( - name="Enable sudo log", - command="sudo rm /etc/sudoers.d/99-no-syslog", - timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, - env={}, - ) + if base == BaseImage.RESOLUTE: + enable_sudo_log_cmd = Command( + name="Enable sudo-rs log", + command="sudo rm /etc/rsyslog.d/00-nosudo.conf", + timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, + env={}, + ) + else: + enable_sudo_log_cmd = Command( + name="Enable sudo log", + command="sudo rm /etc/sudoers.d/99-no-syslog", + timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, + env={}, + ) try: for cmd in ( diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index bbc00941..947704e6 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -96,14 +96,13 @@ class Commands: name="journal does not contain external script secrets", command="! journalctl | grep 'MISSING'", ), - # The sudo-rs in 26.04 does not work with 'Defaults !syslog' Commands( name="journal does not contain external script url", - command=f"lsb_release -r | grep 26.04 || ! journalctl | grep '{TESTDATA_TEST_SCRIPT_URL}'", + command=f"! journalctl | grep '{TESTDATA_TEST_SCRIPT_URL}'", ), Commands( name="journal does not contain script content", - command="lsb_release -r | grep 26.04 || ! journalctl | grep '/home/ubuntu/secret.txt'", + command="! journalctl | grep '/home/ubuntu/secret.txt'", ), Commands( name="/var/log/auth.logs does not contain external script secrets", From 48c4869e0eefb131afee4604da0a4fed6986f465 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 17 Mar 2026 10:28:51 +0800 Subject: [PATCH 54/61] Test disable sudo-rs logs --- app/src/github_runner_image_builder/openstack_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index d7278fb2..e56d0754 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -629,7 +629,7 @@ def _execute_external_script( if base == BaseImage.RESOLUTE: disable_sudo_log_cmd = Command( name="Disable sudo-rs log", - command="echo ':programname, isequal, \"sudo-rs\" ~' | sudo tee /etc/rsyslog.d/00-nosudo.conf", + command="echo ':programname, isequal, \"sudo\" ~' | sudo tee /etc/rsyslog.d/00-nosudo.conf", timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, env={}, ) From 74016b03edf7a337f419316c3f3516749053f595 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 17 Mar 2026 11:57:29 +0800 Subject: [PATCH 55/61] Test another way to disable sudo-rs logging --- .../openstack_builder.py | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index e56d0754..906e5020 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -629,7 +629,7 @@ def _execute_external_script( if base == BaseImage.RESOLUTE: disable_sudo_log_cmd = Command( name="Disable sudo-rs log", - command="echo ':programname, isequal, \"sudo\" ~' | sudo tee /etc/rsyslog.d/00-nosudo.conf", + command="echo 'Defaults syslog=none' | sudo tee /etc/sudoers.d/99-no-syslog", timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, env={}, ) @@ -659,20 +659,12 @@ def _execute_external_script( timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, env={}, ) - if base == BaseImage.RESOLUTE: - enable_sudo_log_cmd = Command( - name="Enable sudo-rs log", - command="sudo rm /etc/rsyslog.d/00-nosudo.conf", - timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, - env={}, - ) - else: - enable_sudo_log_cmd = Command( - name="Enable sudo log", - command="sudo rm /etc/sudoers.d/99-no-syslog", - timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, - env={}, - ) + enable_sudo_log_cmd = Command( + name="Enable sudo log", + command="sudo rm /etc/sudoers.d/99-no-syslog", + timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, + env={}, + ) try: for cmd in ( From 0ca213745b962bb87d99abd5c23a947cbc517cad Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 17 Mar 2026 13:46:43 +0800 Subject: [PATCH 56/61] Test --- app/tests/integration/commands.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 947704e6..73f7b5df 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -96,13 +96,14 @@ class Commands: name="journal does not contain external script secrets", command="! journalctl | grep 'MISSING'", ), + # The sudo-rs in 26.04 cannot be disable with "Defaults !syslog". Commands( name="journal does not contain external script url", - command=f"! journalctl | grep '{TESTDATA_TEST_SCRIPT_URL}'", + command=f"lsb_release -r | grep 26.04 || ! journalctl | grep '{TESTDATA_TEST_SCRIPT_URL}'", ), Commands( name="journal does not contain script content", - command="! journalctl | grep '/home/ubuntu/secret.txt'", + command="lsb_release -r | grep 26.04 || ! journalctl | grep '/home/ubuntu/secret.txt'", ), Commands( name="/var/log/auth.logs does not contain external script secrets", From f96ebb45a2d6a0ae10528e345d5be5e3cd3eb2de Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Tue, 17 Mar 2026 14:55:37 +0800 Subject: [PATCH 57/61] Remove legacy code --- .../openstack_builder.py | 24 ++++++------------- app/tests/integration/commands.py | 8 +++---- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 906e5020..53c9b550 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -343,7 +343,6 @@ def run( script_url=script_url.geturl(), script_secrets=image_config.script_config.script_secrets, ssh_conn=ssh_conn, - base=image_config.base, ) _shutoff_server(conn=conn, server=builder) image = store.create_snapshot( @@ -612,7 +611,7 @@ def _wait_for_cloud_init_complete( def _execute_external_script( - script_url: str, script_secrets: dict[str, str], ssh_conn: fabric.Connection, base: BaseImage, + script_url: str, script_secrets: dict[str, str], ssh_conn: fabric.Connection, ) -> None: """Execute the external script on the OpenStack instance. @@ -620,26 +619,17 @@ def _execute_external_script( script_url: The external script URL to download and execute. script_secrets: The secrets to pass as environment variables to the script. ssh_conn: The SSH connection instance to the OpenStack server instance. - base: The ubuntu base image. Raises: ExternalScriptError: If the external script (or setup/cleanup of it) failed to execute. """ Command = namedtuple("Command", ["name", "command", "timeout", "env"]) - if base == BaseImage.RESOLUTE: - disable_sudo_log_cmd = Command( - name="Disable sudo-rs log", - command="echo 'Defaults syslog=none' | sudo tee /etc/sudoers.d/99-no-syslog", - timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, - env={}, - ) - else: - disable_sudo_log_cmd = Command( - name="Disable sudo log", - command="echo 'Defaults !syslog' | sudo tee /etc/sudoers.d/99-no-syslog", - timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, - env={}, - ) + disable_sudo_log_cmd = Command( + name="Disable sudo log", + command="echo 'Defaults !syslog' | sudo tee /etc/sudoers.d/99-no-syslog", + timeout=EXTERNAL_SCRIPT_GENERAL_TIMEOUT, + env={}, + ) script_setup_cmd = Command( name="Download the external script and set permissions", command=f'sudo curl "{script_url}" -o {EXTERNAL_SCRIPT_PATH} ' diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 73f7b5df..ec767b21 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -61,7 +61,7 @@ class Commands: ), # Exclude 26.04 from HWE kernel test since there is no HWE kernel for it. Commands( - name="test that HWE kernel is installed", + name="test that HWE kernel is installed (only for non-resolute)", command="lsb_release -r | grep 26.04 || uname -a | " "grep $(dpkg -l | grep linux-generic-hwe | awk '{print $3}' | cut -d'.' -f1-3)", ), @@ -70,7 +70,7 @@ class Commands: command="sudo sysctl -a | grep 'net.core.default_qdisc = fq'", ), Commands( - name="test network congestion policy", + name="test network congestion policy (only for non-resolute)", command="lsb_release -r | grep 26.04 || " "sudo sysctl -a | grep 'net.ipv4.tcp_congestion_control = bbr'", ), @@ -98,11 +98,11 @@ class Commands: ), # The sudo-rs in 26.04 cannot be disable with "Defaults !syslog". Commands( - name="journal does not contain external script url", + name="journal does not contain external script url (only for non-resolute)", command=f"lsb_release -r | grep 26.04 || ! journalctl | grep '{TESTDATA_TEST_SCRIPT_URL}'", ), Commands( - name="journal does not contain script content", + name="journal does not contain script content (only for non-resolute)", command="lsb_release -r | grep 26.04 || ! journalctl | grep '/home/ubuntu/secret.txt'", ), Commands( From fcd97d756c9beea3ef4e03920eeb153464f157d0 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 18 Mar 2026 07:32:39 +0800 Subject: [PATCH 58/61] Test --- app/tests/integration/commands.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index ec767b21..6a8fdcd3 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -113,12 +113,13 @@ class Commands: name="/var/log/auth.logs does not contain external script secrets", command="! grep 'MISSING' /var/log/auth.log*", ), + # The sudo-rs in 26.04 cannot be disable with "Defaults !syslog". Commands( name="/var/log/auth.logs does not contain external script url", - command=f"! grep '{TESTDATA_TEST_SCRIPT_URL}' /var/log/auth.log*", + command=f"lsb_release -r | grep 26.04 || ! grep '{TESTDATA_TEST_SCRIPT_URL}' /var/log/auth.log*", ), Commands( name="/var/log/auth.logs does not contain script content", - command="! grep '/home/ubuntu/secret.txt' /var/log/auth.log*", + command="lsb_release -r | grep 26.04 || ! grep '/home/ubuntu/secret.txt' /var/log/auth.log*", ), ) From 2f9b7db19df4d7f799da93ad7e235ddd044efd34 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Wed, 18 Mar 2026 07:36:40 +0800 Subject: [PATCH 59/61] Fix comments --- .../github_runner_image_builder/openstack_builder.py | 4 +++- .../templates/cloud-init.sh.j2 | 2 +- app/tests/integration/commands.py | 10 ++++++---- app/tests/unit/test_openstack_builder.py | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/app/src/github_runner_image_builder/openstack_builder.py b/app/src/github_runner_image_builder/openstack_builder.py index 53c9b550..36cc7f63 100644 --- a/app/src/github_runner_image_builder/openstack_builder.py +++ b/app/src/github_runner_image_builder/openstack_builder.py @@ -611,7 +611,9 @@ def _wait_for_cloud_init_complete( def _execute_external_script( - script_url: str, script_secrets: dict[str, str], ssh_conn: fabric.Connection, + script_url: str, + script_secrets: dict[str, str], + ssh_conn: fabric.Connection, ) -> None: """Execute the external script on the OpenStack instance. diff --git a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 index 92575209..2b4ebcfa 100644 --- a/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 +++ b/app/src/github_runner_image_builder/templates/cloud-init.sh.j2 @@ -72,7 +72,7 @@ function install_apt_packages() { DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get upgrade -y echo "Installing apt packages $packages" DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install -y --no-install-recommends ${packages} - # Exclude resolute from HWE kernel test since there is no HWE kernel for it. + # Skip installing the HWE kernel package on resolute since there is no HWE kernel for it. if [ $RELEASE != "resolute" ]; then echo "Installing linux-generic-hwe-${hwe_version}" DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install -y --install-recommends linux-generic-hwe-${hwe_version} diff --git a/app/tests/integration/commands.py b/app/tests/integration/commands.py index 6a8fdcd3..9b327140 100644 --- a/app/tests/integration/commands.py +++ b/app/tests/integration/commands.py @@ -96,7 +96,7 @@ class Commands: name="journal does not contain external script secrets", command="! journalctl | grep 'MISSING'", ), - # The sudo-rs in 26.04 cannot be disable with "Defaults !syslog". + # The sudo-rs in 26.04 cannot be disabled with "Defaults !syslog". Commands( name="journal does not contain external script url (only for non-resolute)", command=f"lsb_release -r | grep 26.04 || ! journalctl | grep '{TESTDATA_TEST_SCRIPT_URL}'", @@ -113,13 +113,15 @@ class Commands: name="/var/log/auth.logs does not contain external script secrets", command="! grep 'MISSING' /var/log/auth.log*", ), - # The sudo-rs in 26.04 cannot be disable with "Defaults !syslog". + # The sudo-rs in 26.04 cannot be disabled with "Defaults !syslog". Commands( name="/var/log/auth.logs does not contain external script url", - command=f"lsb_release -r | grep 26.04 || ! grep '{TESTDATA_TEST_SCRIPT_URL}' /var/log/auth.log*", + command="lsb_release -r | grep 26.04 || " + f"! grep '{TESTDATA_TEST_SCRIPT_URL}' /var/log/auth.log*", ), Commands( name="/var/log/auth.logs does not contain script content", - command="lsb_release -r | grep 26.04 || ! grep '/home/ubuntu/secret.txt' /var/log/auth.log*", + command="lsb_release -r | grep 26.04 || " + "! grep '/home/ubuntu/secret.txt' /var/log/auth.log*", ), ) diff --git a/app/tests/unit/test_openstack_builder.py b/app/tests/unit/test_openstack_builder.py index 73b1c0a8..97350053 100644 --- a/app/tests/unit/test_openstack_builder.py +++ b/app/tests/unit/test_openstack_builder.py @@ -783,7 +783,7 @@ def test__generate_cloud_init_script( DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get upgrade -y echo "Installing apt packages $packages" DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install -y --no-install-recommends ${{packages}} - # Exclude resolute from HWE kernel test since there is no HWE kernel for it. + # Skip installing the HWE kernel package on resolute since there is no HWE kernel for it. if [ $RELEASE != "resolute" ]; then echo "Installing linux-generic-hwe-${{hwe_version}}" DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install -y --install-recommends linux-generic-hwe-${{hwe_version}} From 81517a47eb9c662916e5e694fd7aef5a4867d59b Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Fri, 20 Mar 2026 12:13:38 +0800 Subject: [PATCH 60/61] Fix missing resolute in charm config --- src/state.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/state.py b/src/state.py index 0cc369cd..506d68f2 100644 --- a/src/state.py +++ b/src/state.py @@ -137,11 +137,13 @@ class BaseImage(str, Enum): FOCAL: The focal ubuntu LTS image. JAMMY: The jammy ubuntu LTS image. NOBLE: The noble ubuntu LTS image. + RESOLUTE: The resolute ubuntu LTS image. """ FOCAL = "focal" JAMMY = "jammy" NOBLE = "noble" + RESOLUTE = "resolute" def __str__(self) -> str: """Interpolate to string value. From e1681490ba7fc9ca4cadb7b426f3f70ac3b70fb0 Mon Sep 17 00:00:00 2001 From: yhaliaw Date: Fri, 20 Mar 2026 12:15:43 +0800 Subject: [PATCH 61/61] Add changelog --- docs/changelog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index f14883ad..e0367e10 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,6 +1,10 @@ +## [#206 Add resolute image support] + +* Add resolute image support. + ## [#198 Update integration tests](https://github.com/canonical/github-runner-image-builder-operator/pull/198) (2026-02-17) * Update integration tests that were intentionally not done in #185.