[Packaging] Azure CLI container images and RPMs based on Azure Linux 4.0

azure-pipelines.yml

@@ -38,6 +38,20 @@ parameters:
     value: arm64
     pool: pool-ubuntu-latest-arm64
 
+# Azure Linux versions to build in parallel. AZL3 is the current stable release;
+# AZL4 is being onboarded (currently beta) to pick up the patched OpenSSL and
+# resolve the S360 vulnerability. Both are built and published side-by-side
+# until AZL4 reaches GA and AZL3 is deprecated.
+- name: azureLinuxVersions
+  type: object
+  default:
+  - name: '3.0'
+    tag: '3.0'
+    image: mcr.microsoft.com/azurelinux/base/core:3.0
+  - name: '4.0 (Beta)'
+    tag: '4.0'
+    image: mcr.microsoft.com/azurelinux-beta/base/core:4.0
+
 jobs:
 - job: CheckPullRequest
   displayName: "Check the Format of Pull Request Title and Content"
@@ -351,13 +365,14 @@ jobs:
   dependsOn: BuildRpmPackagesAzureLinux
   strategy:
     matrix:
-      ${{ each arch in parameters.architectures }}:
-        Azure Linux 3.0 ${{ arch.name }}:
-          pool: ${{ arch.pool }}
-          artifactName: docker-azurelinux3.0-${{ arch.value }}
-          dockerfile: azure-linux.dockerfile
-          packageArtifactName: rpm-azurelinux3.0-${{ arch.value }}
-          image: mcr.microsoft.com/azurelinux/base/core:3.0
+      ${{ each azl in parameters.azureLinuxVersions }}:
+        ${{ each arch in parameters.architectures }}:
+          Azure Linux ${{ azl.name }} ${{ arch.name }}:
+            pool: ${{ arch.pool }}
+            artifactName: docker-azurelinux${{ azl.tag }}-${{ arch.value }}
+            dockerfile: azure-linux.dockerfile
+            packageArtifactName: rpm-azurelinux${{ azl.tag }}-${{ arch.value }}
+            image: ${{ azl.image }}
   pool:
     name: $(pool)
   steps:
@@ -395,10 +410,11 @@ jobs:
     - ExtractMetadata
   strategy:
     matrix:
-      ${{ each arch in parameters.architectures }}:
-        Azure Linux 3.0 ${{ arch.name }}:
-          pool: ${{ arch.pool }}
-          artifactName: docker-azurelinux3.0-${{ arch.value }}
+      ${{ each azl in parameters.azureLinuxVersions }}:
+        ${{ each arch in parameters.architectures }}:
+          Azure Linux ${{ azl.name }} ${{ arch.name }}:
+            pool: ${{ arch.pool }}
+            artifactName: docker-azurelinux${{ azl.tag }}-${{ arch.value }}
   pool:
     name: $(pool)
   steps:
@@ -775,11 +791,12 @@ jobs:
   condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule'))
   strategy:
     matrix:
-      ${{ each arch in parameters.architectures }}:
-        Azure Linux 3.0 ${{ arch.name }}:
-          image: mcr.microsoft.com/azurelinux/base/core:3.0
-          artifact: rpm-azurelinux3.0-${{ arch.value }}
-          pool: ${{ arch.pool }}
+      ${{ each azl in parameters.azureLinuxVersions }}:
+        ${{ each arch in parameters.architectures }}:
+          Azure Linux ${{ azl.name }} ${{ arch.name }}:
+            image: ${{ azl.image }}
+            artifact: rpm-azurelinux${{ azl.tag }}-${{ arch.value }}
+            pool: ${{ arch.pool }}
   pool:
     name: $(pool)
   steps:
@@ -821,11 +838,12 @@ jobs:
     name: $(pool)
   strategy:
     matrix:
-      ${{ each arch in parameters.architectures }}:
-        Azure Linux 3.0 ${{ arch.name }}:
-          image: mcr.microsoft.com/azurelinux/base/core:3.0
-          artifact: rpm-azurelinux3.0-${{ arch.value }}
-          pool: ${{ arch.pool }}
+      ${{ each azl in parameters.azureLinuxVersions }}:
+        ${{ each arch in parameters.architectures }}:
+          Azure Linux ${{ azl.name }} ${{ arch.name }}:
+            image: ${{ azl.image }}
+            artifact: rpm-azurelinux${{ azl.tag }}-${{ arch.value }}
+            pool: ${{ arch.pool }}
   steps:
   - task: DownloadPipelineArtifact@1
     displayName: 'Download Metadata'

scripts/release/rpm/README.md

@@ -17,7 +17,12 @@ docker build --target build-env -f ./scripts/release/rpm/fedora.dockerfile -t az
 _Azure Linux:_
 
 ```bash
+# Build against Azure Linux 4.0 (Beta) - the default in the dockerfile
 docker build --target build-env -f ./scripts/release/rpm/azurelinux.dockerfile -t azure/azure-cli:azurelinux-builder .
+
+# Or build against Azure Linux 3.0 explicitly
+docker build --target build-env --build-arg image=mcr.microsoft.com/azurelinux/base/core:3.0 \
+    -f ./scripts/release/rpm/azurelinux.dockerfile -t azure/azure-cli:azurelinux3-builder .
 ```
 
 After several minutes, this will have created a Docker image named `azure/azure-cli:centos7-builder` containing an
@@ -36,7 +41,11 @@ docker run azure/azure-cli:fedora29-builder cat /root/rpmbuild/RPMS/x86_64/azure
 
 _Azure Linux:_
 ```bash
-docker run azure/azure-cli:azurelinux-builder cat /usr/src/azl/RPMS/x86_64/azure-cli-dev-1.azl3.x86_64.rpm > ./bin/azure-cli-dev-1.azl3.x86_64.rpm
+# AZL4 output path (rpmbuild default topdir is /root/rpmbuild)
+docker run azure/azure-cli:azurelinux-builder cat /root/rpmbuild/RPMS/x86_64/azure-cli-dev-1.azl4.x86_64.rpm > ./bin/azure-cli-dev-1.azl4.x86_64.rpm
+
+# AZL3 output path (rpmbuild default topdir is /usr/src/azl)
+docker run azure/azure-cli:azurelinux3-builder cat /usr/src/azl/RPMS/x86_64/azure-cli-dev-1.azl3.x86_64.rpm > ./bin/azure-cli-dev-1.azl3.x86_64.rpm
 ```
 
 This launches a container running from the image built and tagged by the previous command, prints the contents of the

scripts/release/rpm/azurelinux.dockerfile

@@ -1,4 +1,6 @@
-ARG image=mcr.microsoft.com/azurelinux/base/core:3.0
+# Default is Azure Linux 4.0 (Beta), but this dockerfile is shared with Azure Linux 3.0
+# builds too -- the CI pipeline always overrides this with --build-arg image=<AZL3 or AZL4>.
+ARG image=mcr.microsoft.com/azurelinux-beta/base/core:4.0
 
 FROM ${image} AS build-env
 ARG cli_version=dev
@@ -16,15 +18,21 @@ WORKDIR /azure-cli
 
 COPY . .
 
-# Azure Linux 3's python3 is 3.12, the rpm paths are
-#   /usr/src/azl/RPMS/x86_64/azure-cli-2.63.0-1.azl3.x86_64.rpm
-#   /usr/src/azl/RPMS/aarch64/azure-cli-2.63.0-1.azl3.aarch64.rpm
+# This dockerfile is shared by both Azure Linux 3.0 and Azure Linux 4.0 (Beta) builds
+# (the ${image} build-arg selects which base image is used). The two base images use
+# different rpmbuild topdir defaults, so the built RPM ends up in different locations:
+#   AZL3: /usr/src/azl/RPMS/x86_64/azure-cli-2.63.0-1.azl3.x86_64.rpm
+#   AZL4: /root/rpmbuild/RPMS/x86_64/azure-cli-2.63.0-1.azl4.x86_64.rpm
+# Use `find` so the same RUN step works for either base image without needing to know
+# which topdir it used.
 RUN --mount=type=secret,id=PIP_INDEX_URL export PIP_INDEX_URL=$(cat /run/secrets/PIP_INDEX_URL) && \
     dos2unix ./scripts/release/rpm/azure-cli.spec && \
     REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=python3 PYTHON_CMD=python3 \
     rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \
-    cp /usr/src/*/RPMS/*/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \
-    mkdir /out && cp /usr/src/*/RPMS/*/azure-cli-${cli_version}-1.*.rpm /out/
+    RPM_PATH=$(find /usr/src/azl/RPMS /root/rpmbuild/RPMS -type f -name "azure-cli-${cli_version}-1.*.rpm" 2>/dev/null | head -n 1) && \
+    if [ -z "$RPM_PATH" ]; then echo "ERROR: No RPM found in expected directories. Contents:"; ls -la /usr/src/azl/RPMS 2>/dev/null || echo "  /usr/src/azl/RPMS not found"; ls -la /root/rpmbuild/RPMS 2>/dev/null || echo "  /root/rpmbuild/RPMS not found"; exit 1; fi && \
+    cp "$RPM_PATH" /azure-cli-dev.rpm && \
+    mkdir /out && cp "$RPM_PATH" /out/
 
 FROM ${image} AS execution-env
 

scripts/release/rpm/pipeline_azurelinux.sh

@@ -6,7 +6,10 @@ set -exv
 
 : "${BUILD_STAGINGDIRECTORY:?BUILD_STAGINGDIRECTORY environment variable not set.}"
 
-# IMAGE should be Azure Linux docker image url, such as mcr.microsoft.com/azurelinux/base/core:3.0
+# IMAGE should be Azure Linux docker image url. The CI pipeline runs this script once per
+# matrix row (Azure Linux 3.0 and Azure Linux 4.0 Beta, each x86_64/aarch64), passing in the
+# appropriate image, e.g. mcr.microsoft.com/azurelinux/base/core:3.0 or
+# mcr.microsoft.com/azurelinux-beta/base/core:4.0
 : "${IMAGE:?IMAGE environment variable not set.}"
 
 CLI_VERSION=`cat src/azure-cli/azure/cli/__main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// |  sed s/\"//g`

src/azure-cli-core/azure/cli/core/extension/operations.py

@@ -180,7 +180,7 @@ def _add_whl_ext(cli_ctx, source, ext_sha256=None, pip_extra_index_urls=None, pi
     return extension_name
 
 
-def _install_deps_for_psycopg2():  # pylint: disable=too-many-statements
+def _install_deps_for_psycopg2():  # pylint: disable=too-many-statements, too-many-branches
     # If we are in Cloud Shell, dependencies should have already been installed.
     from azure.cli.core.util import in_cloud_console
     if in_cloud_console():
@@ -215,8 +215,17 @@ def _install_deps_for_psycopg2():  # pylint: disable=too-many-statements
                 '/usr/local/opt/openssl/lib/'
             ])
     elif system == 'Linux':
-        distname, _ = get_linux_distro()
-        distname = distname.lower().strip()
+        distname, distversion = get_linux_distro()
+        distname = (distname or '').lower().strip()
+        distversion = (distversion or '').strip()
+        # Extract major version from VERSION_ID (e.g., "4", "4.0", "4.0.1" all map to major=4)
+        azl_major_version = None
+        if 'azure linux' in distname and distversion:
+            try:
+                azl_major_version = int(distversion.partition('.')[0])
+            except ValueError:
+                pass
+        is_azure_linux_4 = azl_major_version == 4
         if installer == 'DEB' or any(x in distname for x in ['ubuntu', 'debian']):
             exit_code = subprocess.call(['dpkg', '-s', 'gcc', 'libpq-dev', 'python3-dev'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
             if exit_code != 0:
@@ -230,7 +239,7 @@ def _install_deps_for_psycopg2():  # pylint: disable=too-many-statements
                 if exit_code == 0:
                     logger.debug("Install dependencies with '%s'", " ".join(apt_install_cmd))
                     subprocess.call(apt_install_cmd, True)
-        elif installer == 'RPM' or any(x in distname for x in ['centos', 'rhel', 'red hat', 'fedora', 'opensuse', 'suse', 'sles']):
+        elif installer == 'RPM' or any(x in distname for x in ['centos', 'rhel', 'red hat', 'fedora', 'opensuse', 'suse', 'sles']) or is_azure_linux_4:
             if any(x in distname for x in ['centos', 'rhel', 'red hat', 'fedora']):
                 yum_install_cmd = 'yum install -y gcc postgresql-devel python3-devel'.split()
                 if os.geteuid() != 0:  # pylint: disable=no-member
@@ -249,6 +258,29 @@ def _install_deps_for_psycopg2():  # pylint: disable=too-many-statements
                 if exit_code == 0:
                     logger.debug("Install dependencies with '%s'", " ".join(zypper_install_cmd))
                     subprocess.call(zypper_install_cmd)
+            elif is_azure_linux_4:
+                from shutil import which
+                if which('tdnf') is None:
+                    raise CLIError(
+                        'tdnf package manager not found. This extension requires Azure Linux 4 with tdnf '
+                        'to install required system dependencies: gcc, libpq-devel, python3-devel, binutils, '
+                        'glibc-devel, kernel-headers.'
+                    )
+                rpm_packages = ['gcc', 'libpq-devel', 'python3-devel', 'binutils', 'glibc-devel', 'kernel-headers']
+                rpm_install_cmd = ['tdnf', 'install', '-y'] + rpm_packages
+                if os.geteuid() != 0:  # pylint: disable=no-member
+                    rpm_install_cmd.insert(0, 'sudo')
+                logger.debug("Install dependencies with '%s'", " ".join(rpm_install_cmd))
+                logger.warning(
+                    'This extension depends on %s and will now be installed.',
+                    ' '.join(rpm_packages)
+                )
+                exit_code = subprocess.call(rpm_install_cmd)
+                if exit_code != 0:
+                    raise CLIError(
+                        'Failed to install required system dependencies for psycopg2: {}.'
+                        .format(' '.join(rpm_packages))
+                    )
         elif installer == 'DOCKER' or any(x in distname for x in ['alpine linux']):
             apk_install_cmd = 'apk add --no-cache libpq-dev'.split()
             logger.debug("Install dependencies with '%s'", " ".join(apk_install_cmd))

src/azure-cli-core/azure/cli/core/extension/tests/latest/test_extension_commands.py

@@ -14,7 +14,8 @@
 from azure.cli.core.extension import get_extension, build_extension_path
 from azure.cli.core.extension.operations import (add_extension_to_path, list_extensions, add_extension,
                                                  show_extension, remove_extension, update_extension,
-                                                 list_available_extensions, OUT_KEY_NAME, OUT_KEY_VERSION,
+                                                 list_available_extensions, _install_deps_for_psycopg2,
+                                                 OUT_KEY_NAME, OUT_KEY_VERSION,
                                                  OUT_KEY_METADATA, OUT_KEY_PATH)
 from azure.cli.core.extension._resolve import NoExtensionCandidatesError
 from azure.cli.core.mock import DummyCli
@@ -481,6 +482,68 @@ def test_add_extension_azure_to_path(self):
         self.assertSequenceEqual(old_path_1, list(azure.__path__))
         self.assertSequenceEqual(old_path_2, list(azure.mgmt.__path__))
 
+    def test_install_psycopg2_deps_uses_azl4_packages(self):
+        with mock.patch('azure.cli.core.util.in_cloud_console', return_value=False), \
+                mock.patch('platform.system', return_value='Linux'), \
+                mock.patch('azure.cli.core.util.get_linux_distro', return_value=('Azure Linux', '4.0')), \
+                mock.patch.dict('os.environ', {'AZ_INSTALLER': 'RPM'}), \
+                mock.patch('shutil.which', side_effect=lambda cmd: '/usr/bin/tdnf' if cmd == 'tdnf' else None), \
+                mock.patch('os.geteuid', return_value=0, create=True), \
+                mock.patch('subprocess.call', return_value=0) as subprocess_call:
+            _install_deps_for_psycopg2()
+
+        azure_linux_packages = ['gcc', 'libpq-devel', 'python3-devel', 'binutils', 'glibc-devel', 'kernel-headers']
+        subprocess_call.assert_any_call(['tdnf', 'install', '-y'] + azure_linux_packages)
+        # Ensure no yum/sudo yum calls were made
+        for call in subprocess_call.call_args_list:
+            args = call.args[0] if call.args else []
+            self.assertFalse(any(cmd in args for cmd in ['yum', 'zypper']),
+                             f"Unexpected package manager call: {args}")
+
+    def test_install_psycopg2_deps_preserves_azl3_rpm_behavior(self):
+        with mock.patch('azure.cli.core.util.in_cloud_console', return_value=False), \
+                mock.patch('platform.system', return_value='Linux'), \
+                mock.patch('azure.cli.core.util.get_linux_distro', return_value=('Azure Linux', '3.0')), \
+                mock.patch.dict('os.environ', {'AZ_INSTALLER': 'RPM'}), \
+                mock.patch('subprocess.call') as subprocess_call:
+            _install_deps_for_psycopg2()
+
+        subprocess_call.assert_not_called()
+
+    def test_install_psycopg2_deps_reports_rpm_install_failure(self):
+        with mock.patch('azure.cli.core.util.in_cloud_console', return_value=False), \
+                mock.patch('platform.system', return_value='Linux'), \
+                mock.patch('azure.cli.core.util.get_linux_distro', return_value=('Azure Linux', '4.0')), \
+                mock.patch.dict('os.environ', {'AZ_INSTALLER': 'RPM'}), \
+                mock.patch('shutil.which', side_effect=lambda cmd: '/usr/bin/tdnf' if cmd == 'tdnf' else None), \
+                mock.patch('os.geteuid', return_value=0, create=True), \
+                mock.patch('subprocess.call', return_value=1):
+            with self.assertRaisesRegex(CLIError, 'Failed to install required system dependencies for psycopg2'):
+                _install_deps_for_psycopg2()
+
+    def test_install_psycopg2_deps_fails_if_tdnf_missing(self):
+        with mock.patch('azure.cli.core.util.in_cloud_console', return_value=False), \
+                mock.patch('platform.system', return_value='Linux'), \
+                mock.patch('azure.cli.core.util.get_linux_distro', return_value=('Azure Linux', '4.0')), \
+                mock.patch.dict('os.environ', {'AZ_INSTALLER': 'RPM'}), \
+                mock.patch('shutil.which', return_value=None):
+            with self.assertRaisesRegex(CLIError, 'tdnf package manager not found'):
+                _install_deps_for_psycopg2()
+
+    def test_install_psycopg2_deps_recognizes_azl4_without_dot_version(self):
+        # Regression test: VERSION_ID may be "4" (no dot) instead of "4.0"
+        with mock.patch('azure.cli.core.util.in_cloud_console', return_value=False), \
+                mock.patch('platform.system', return_value='Linux'), \
+                mock.patch('azure.cli.core.util.get_linux_distro', return_value=('Azure Linux', '4')), \
+                mock.patch.dict('os.environ', {'AZ_INSTALLER': 'RPM'}), \
+                mock.patch('shutil.which', side_effect=lambda cmd: '/usr/bin/tdnf' if cmd == 'tdnf' else None), \
+                mock.patch('os.geteuid', return_value=0, create=True), \
+                mock.patch('subprocess.call', return_value=0) as subprocess_call:
+            _install_deps_for_psycopg2()
+
+        azure_linux_packages = ['gcc', 'libpq-devel', 'python3-devel', 'binutils', 'glibc-devel', 'kernel-headers']
+        subprocess_call.assert_any_call(['tdnf', 'install', '-y'] + azure_linux_packages)
+
     def _setup_cmd(self):
         cmd = mock.MagicMock()
         cmd.cli_ctx = DummyCli()