diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 18fedf3969b..923729483fd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -38,6 +38,20 @@ parameters: value: arm64 pool: pool-ubuntu-latest-arm64 +# Azure Linux versions to build in parallel. AZL3 is the current stable release; +# AZL4 is being onboarded (currently beta) to pick up the patched OpenSSL and +# resolve the S360 vulnerability. Both are built and published side-by-side +# until AZL4 reaches GA and AZL3 is deprecated. +- name: azureLinuxVersions + type: object + default: + - name: '3.0' + tag: '3.0' + image: mcr.microsoft.com/azurelinux/base/core:3.0 + - name: '4.0 (Beta)' + tag: '4.0' + image: mcr.microsoft.com/azurelinux-beta/base/core:4.0 + jobs: - job: CheckPullRequest displayName: "Check the Format of Pull Request Title and Content" @@ -351,13 +365,14 @@ jobs: dependsOn: BuildRpmPackagesAzureLinux strategy: matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - pool: ${{ arch.pool }} - artifactName: docker-azurelinux3.0-${{ arch.value }} - dockerfile: azure-linux.dockerfile - packageArtifactName: rpm-azurelinux3.0-${{ arch.value }} - image: mcr.microsoft.com/azurelinux/base/core:3.0 + ${{ each azl in parameters.azureLinuxVersions }}: + ${{ each arch in parameters.architectures }}: + Azure Linux ${{ azl.name }} ${{ arch.name }}: + pool: ${{ arch.pool }} + artifactName: docker-azurelinux${{ azl.tag }}-${{ arch.value }} + dockerfile: azure-linux.dockerfile + packageArtifactName: rpm-azurelinux${{ azl.tag }}-${{ arch.value }} + image: ${{ azl.image }} pool: name: $(pool) steps: @@ -395,10 +410,11 @@ jobs: - ExtractMetadata strategy: matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - pool: ${{ arch.pool }} - artifactName: docker-azurelinux3.0-${{ arch.value }} + ${{ each azl in parameters.azureLinuxVersions }}: + ${{ each arch in parameters.architectures }}: + Azure Linux ${{ azl.name }} ${{ arch.name }}: + pool: ${{ arch.pool }} + artifactName: docker-azurelinux${{ azl.tag }}-${{ arch.value }} pool: name: $(pool) steps: @@ -775,11 +791,12 @@ jobs: condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) strategy: matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - image: mcr.microsoft.com/azurelinux/base/core:3.0 - artifact: rpm-azurelinux3.0-${{ arch.value }} - pool: ${{ arch.pool }} + ${{ each azl in parameters.azureLinuxVersions }}: + ${{ each arch in parameters.architectures }}: + Azure Linux ${{ azl.name }} ${{ arch.name }}: + image: ${{ azl.image }} + artifact: rpm-azurelinux${{ azl.tag }}-${{ arch.value }} + pool: ${{ arch.pool }} pool: name: $(pool) steps: @@ -821,11 +838,12 @@ jobs: name: $(pool) strategy: matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - image: mcr.microsoft.com/azurelinux/base/core:3.0 - artifact: rpm-azurelinux3.0-${{ arch.value }} - pool: ${{ arch.pool }} + ${{ each azl in parameters.azureLinuxVersions }}: + ${{ each arch in parameters.architectures }}: + Azure Linux ${{ azl.name }} ${{ arch.name }}: + image: ${{ azl.image }} + artifact: rpm-azurelinux${{ azl.tag }}-${{ arch.value }} + pool: ${{ arch.pool }} steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Metadata' diff --git a/scripts/release/rpm/README.md b/scripts/release/rpm/README.md index 9e263d17140..b5d5dee0cf1 100644 --- a/scripts/release/rpm/README.md +++ b/scripts/release/rpm/README.md @@ -17,7 +17,12 @@ docker build --target build-env -f ./scripts/release/rpm/fedora.dockerfile -t az _Azure Linux:_ ```bash +# Build against Azure Linux 4.0 (Beta) - the default in the dockerfile docker build --target build-env -f ./scripts/release/rpm/azurelinux.dockerfile -t azure/azure-cli:azurelinux-builder . + +# Or build against Azure Linux 3.0 explicitly +docker build --target build-env --build-arg image=mcr.microsoft.com/azurelinux/base/core:3.0 \ + -f ./scripts/release/rpm/azurelinux.dockerfile -t azure/azure-cli:azurelinux3-builder . ``` After several minutes, this will have created a Docker image named `azure/azure-cli:centos7-builder` containing an @@ -36,7 +41,11 @@ docker run azure/azure-cli:fedora29-builder cat /root/rpmbuild/RPMS/x86_64/azure _Azure Linux:_ ```bash -docker run azure/azure-cli:azurelinux-builder cat /usr/src/azl/RPMS/x86_64/azure-cli-dev-1.azl3.x86_64.rpm > ./bin/azure-cli-dev-1.azl3.x86_64.rpm +# AZL4 output path (rpmbuild default topdir is /root/rpmbuild) +docker run azure/azure-cli:azurelinux-builder cat /root/rpmbuild/RPMS/x86_64/azure-cli-dev-1.azl4.x86_64.rpm > ./bin/azure-cli-dev-1.azl4.x86_64.rpm + +# AZL3 output path (rpmbuild default topdir is /usr/src/azl) +docker run azure/azure-cli:azurelinux3-builder cat /usr/src/azl/RPMS/x86_64/azure-cli-dev-1.azl3.x86_64.rpm > ./bin/azure-cli-dev-1.azl3.x86_64.rpm ``` This launches a container running from the image built and tagged by the previous command, prints the contents of the diff --git a/scripts/release/rpm/azurelinux.dockerfile b/scripts/release/rpm/azurelinux.dockerfile index fba3af87de6..2e7e65a8ff8 100644 --- a/scripts/release/rpm/azurelinux.dockerfile +++ b/scripts/release/rpm/azurelinux.dockerfile @@ -1,4 +1,6 @@ -ARG image=mcr.microsoft.com/azurelinux/base/core:3.0 +# Default is Azure Linux 4.0 (Beta), but this dockerfile is shared with Azure Linux 3.0 +# builds too -- the CI pipeline always overrides this with --build-arg image=. +ARG image=mcr.microsoft.com/azurelinux-beta/base/core:4.0 FROM ${image} AS build-env ARG cli_version=dev @@ -16,15 +18,21 @@ WORKDIR /azure-cli COPY . . -# Azure Linux 3's python3 is 3.12, the rpm paths are -# /usr/src/azl/RPMS/x86_64/azure-cli-2.63.0-1.azl3.x86_64.rpm -# /usr/src/azl/RPMS/aarch64/azure-cli-2.63.0-1.azl3.aarch64.rpm +# This dockerfile is shared by both Azure Linux 3.0 and Azure Linux 4.0 (Beta) builds +# (the ${image} build-arg selects which base image is used). The two base images use +# different rpmbuild topdir defaults, so the built RPM ends up in different locations: +# AZL3: /usr/src/azl/RPMS/x86_64/azure-cli-2.63.0-1.azl3.x86_64.rpm +# AZL4: /root/rpmbuild/RPMS/x86_64/azure-cli-2.63.0-1.azl4.x86_64.rpm +# Use `find` so the same RUN step works for either base image without needing to know +# which topdir it used. RUN --mount=type=secret,id=PIP_INDEX_URL export PIP_INDEX_URL=$(cat /run/secrets/PIP_INDEX_URL) && \ dos2unix ./scripts/release/rpm/azure-cli.spec && \ REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=python3 PYTHON_CMD=python3 \ rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ - cp /usr/src/*/RPMS/*/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \ - mkdir /out && cp /usr/src/*/RPMS/*/azure-cli-${cli_version}-1.*.rpm /out/ + RPM_PATH=$(find /usr/src/azl/RPMS /root/rpmbuild/RPMS -type f -name "azure-cli-${cli_version}-1.*.rpm" 2>/dev/null | head -n 1) && \ + if [ -z "$RPM_PATH" ]; then echo "ERROR: No RPM found in expected directories. Contents:"; ls -la /usr/src/azl/RPMS 2>/dev/null || echo " /usr/src/azl/RPMS not found"; ls -la /root/rpmbuild/RPMS 2>/dev/null || echo " /root/rpmbuild/RPMS not found"; exit 1; fi && \ + cp "$RPM_PATH" /azure-cli-dev.rpm && \ + mkdir /out && cp "$RPM_PATH" /out/ FROM ${image} AS execution-env diff --git a/scripts/release/rpm/pipeline_azurelinux.sh b/scripts/release/rpm/pipeline_azurelinux.sh index 09bffff50b2..e3eb6d7fe8d 100644 --- a/scripts/release/rpm/pipeline_azurelinux.sh +++ b/scripts/release/rpm/pipeline_azurelinux.sh @@ -6,7 +6,10 @@ set -exv : "${BUILD_STAGINGDIRECTORY:?BUILD_STAGINGDIRECTORY environment variable not set.}" -# IMAGE should be Azure Linux docker image url, such as mcr.microsoft.com/azurelinux/base/core:3.0 +# IMAGE should be Azure Linux docker image url. The CI pipeline runs this script once per +# matrix row (Azure Linux 3.0 and Azure Linux 4.0 Beta, each x86_64/aarch64), passing in the +# appropriate image, e.g. mcr.microsoft.com/azurelinux/base/core:3.0 or +# mcr.microsoft.com/azurelinux-beta/base/core:4.0 : "${IMAGE:?IMAGE environment variable not set.}" CLI_VERSION=`cat src/azure-cli/azure/cli/__main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` diff --git a/src/azure-cli-core/azure/cli/core/extension/operations.py b/src/azure-cli-core/azure/cli/core/extension/operations.py index e20aed58b03..617f8d86a26 100644 --- a/src/azure-cli-core/azure/cli/core/extension/operations.py +++ b/src/azure-cli-core/azure/cli/core/extension/operations.py @@ -180,7 +180,7 @@ def _add_whl_ext(cli_ctx, source, ext_sha256=None, pip_extra_index_urls=None, pi return extension_name -def _install_deps_for_psycopg2(): # pylint: disable=too-many-statements +def _install_deps_for_psycopg2(): # pylint: disable=too-many-statements, too-many-branches # If we are in Cloud Shell, dependencies should have already been installed. from azure.cli.core.util import in_cloud_console if in_cloud_console(): @@ -215,8 +215,17 @@ def _install_deps_for_psycopg2(): # pylint: disable=too-many-statements '/usr/local/opt/openssl/lib/' ]) elif system == 'Linux': - distname, _ = get_linux_distro() - distname = distname.lower().strip() + distname, distversion = get_linux_distro() + distname = (distname or '').lower().strip() + distversion = (distversion or '').strip() + # Extract major version from VERSION_ID (e.g., "4", "4.0", "4.0.1" all map to major=4) + azl_major_version = None + if 'azure linux' in distname and distversion: + try: + azl_major_version = int(distversion.partition('.')[0]) + except ValueError: + pass + is_azure_linux_4 = azl_major_version == 4 if installer == 'DEB' or any(x in distname for x in ['ubuntu', 'debian']): exit_code = subprocess.call(['dpkg', '-s', 'gcc', 'libpq-dev', 'python3-dev'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) if exit_code != 0: @@ -230,7 +239,7 @@ def _install_deps_for_psycopg2(): # pylint: disable=too-many-statements if exit_code == 0: logger.debug("Install dependencies with '%s'", " ".join(apt_install_cmd)) subprocess.call(apt_install_cmd, True) - elif installer == 'RPM' or any(x in distname for x in ['centos', 'rhel', 'red hat', 'fedora', 'opensuse', 'suse', 'sles']): + elif installer == 'RPM' or any(x in distname for x in ['centos', 'rhel', 'red hat', 'fedora', 'opensuse', 'suse', 'sles']) or is_azure_linux_4: if any(x in distname for x in ['centos', 'rhel', 'red hat', 'fedora']): yum_install_cmd = 'yum install -y gcc postgresql-devel python3-devel'.split() if os.geteuid() != 0: # pylint: disable=no-member @@ -249,6 +258,29 @@ def _install_deps_for_psycopg2(): # pylint: disable=too-many-statements if exit_code == 0: logger.debug("Install dependencies with '%s'", " ".join(zypper_install_cmd)) subprocess.call(zypper_install_cmd) + elif is_azure_linux_4: + from shutil import which + if which('tdnf') is None: + raise CLIError( + 'tdnf package manager not found. This extension requires Azure Linux 4 with tdnf ' + 'to install required system dependencies: gcc, libpq-devel, python3-devel, binutils, ' + 'glibc-devel, kernel-headers.' + ) + rpm_packages = ['gcc', 'libpq-devel', 'python3-devel', 'binutils', 'glibc-devel', 'kernel-headers'] + rpm_install_cmd = ['tdnf', 'install', '-y'] + rpm_packages + if os.geteuid() != 0: # pylint: disable=no-member + rpm_install_cmd.insert(0, 'sudo') + logger.debug("Install dependencies with '%s'", " ".join(rpm_install_cmd)) + logger.warning( + 'This extension depends on %s and will now be installed.', + ' '.join(rpm_packages) + ) + exit_code = subprocess.call(rpm_install_cmd) + if exit_code != 0: + raise CLIError( + 'Failed to install required system dependencies for psycopg2: {}.' + .format(' '.join(rpm_packages)) + ) elif installer == 'DOCKER' or any(x in distname for x in ['alpine linux']): apk_install_cmd = 'apk add --no-cache libpq-dev'.split() logger.debug("Install dependencies with '%s'", " ".join(apk_install_cmd)) diff --git a/src/azure-cli-core/azure/cli/core/extension/tests/latest/test_extension_commands.py b/src/azure-cli-core/azure/cli/core/extension/tests/latest/test_extension_commands.py index 2f81268935d..13563d5cab9 100644 --- a/src/azure-cli-core/azure/cli/core/extension/tests/latest/test_extension_commands.py +++ b/src/azure-cli-core/azure/cli/core/extension/tests/latest/test_extension_commands.py @@ -14,7 +14,8 @@ from azure.cli.core.extension import get_extension, build_extension_path from azure.cli.core.extension.operations import (add_extension_to_path, list_extensions, add_extension, show_extension, remove_extension, update_extension, - list_available_extensions, OUT_KEY_NAME, OUT_KEY_VERSION, + list_available_extensions, _install_deps_for_psycopg2, + OUT_KEY_NAME, OUT_KEY_VERSION, OUT_KEY_METADATA, OUT_KEY_PATH) from azure.cli.core.extension._resolve import NoExtensionCandidatesError from azure.cli.core.mock import DummyCli @@ -481,6 +482,68 @@ def test_add_extension_azure_to_path(self): self.assertSequenceEqual(old_path_1, list(azure.__path__)) self.assertSequenceEqual(old_path_2, list(azure.mgmt.__path__)) + def test_install_psycopg2_deps_uses_azl4_packages(self): + with mock.patch('azure.cli.core.util.in_cloud_console', return_value=False), \ + mock.patch('platform.system', return_value='Linux'), \ + mock.patch('azure.cli.core.util.get_linux_distro', return_value=('Azure Linux', '4.0')), \ + mock.patch.dict('os.environ', {'AZ_INSTALLER': 'RPM'}), \ + mock.patch('shutil.which', side_effect=lambda cmd: '/usr/bin/tdnf' if cmd == 'tdnf' else None), \ + mock.patch('os.geteuid', return_value=0, create=True), \ + mock.patch('subprocess.call', return_value=0) as subprocess_call: + _install_deps_for_psycopg2() + + azure_linux_packages = ['gcc', 'libpq-devel', 'python3-devel', 'binutils', 'glibc-devel', 'kernel-headers'] + subprocess_call.assert_any_call(['tdnf', 'install', '-y'] + azure_linux_packages) + # Ensure no yum/sudo yum calls were made + for call in subprocess_call.call_args_list: + args = call.args[0] if call.args else [] + self.assertFalse(any(cmd in args for cmd in ['yum', 'zypper']), + f"Unexpected package manager call: {args}") + + def test_install_psycopg2_deps_preserves_azl3_rpm_behavior(self): + with mock.patch('azure.cli.core.util.in_cloud_console', return_value=False), \ + mock.patch('platform.system', return_value='Linux'), \ + mock.patch('azure.cli.core.util.get_linux_distro', return_value=('Azure Linux', '3.0')), \ + mock.patch.dict('os.environ', {'AZ_INSTALLER': 'RPM'}), \ + mock.patch('subprocess.call') as subprocess_call: + _install_deps_for_psycopg2() + + subprocess_call.assert_not_called() + + def test_install_psycopg2_deps_reports_rpm_install_failure(self): + with mock.patch('azure.cli.core.util.in_cloud_console', return_value=False), \ + mock.patch('platform.system', return_value='Linux'), \ + mock.patch('azure.cli.core.util.get_linux_distro', return_value=('Azure Linux', '4.0')), \ + mock.patch.dict('os.environ', {'AZ_INSTALLER': 'RPM'}), \ + mock.patch('shutil.which', side_effect=lambda cmd: '/usr/bin/tdnf' if cmd == 'tdnf' else None), \ + mock.patch('os.geteuid', return_value=0, create=True), \ + mock.patch('subprocess.call', return_value=1): + with self.assertRaisesRegex(CLIError, 'Failed to install required system dependencies for psycopg2'): + _install_deps_for_psycopg2() + + def test_install_psycopg2_deps_fails_if_tdnf_missing(self): + with mock.patch('azure.cli.core.util.in_cloud_console', return_value=False), \ + mock.patch('platform.system', return_value='Linux'), \ + mock.patch('azure.cli.core.util.get_linux_distro', return_value=('Azure Linux', '4.0')), \ + mock.patch.dict('os.environ', {'AZ_INSTALLER': 'RPM'}), \ + mock.patch('shutil.which', return_value=None): + with self.assertRaisesRegex(CLIError, 'tdnf package manager not found'): + _install_deps_for_psycopg2() + + def test_install_psycopg2_deps_recognizes_azl4_without_dot_version(self): + # Regression test: VERSION_ID may be "4" (no dot) instead of "4.0" + with mock.patch('azure.cli.core.util.in_cloud_console', return_value=False), \ + mock.patch('platform.system', return_value='Linux'), \ + mock.patch('azure.cli.core.util.get_linux_distro', return_value=('Azure Linux', '4')), \ + mock.patch.dict('os.environ', {'AZ_INSTALLER': 'RPM'}), \ + mock.patch('shutil.which', side_effect=lambda cmd: '/usr/bin/tdnf' if cmd == 'tdnf' else None), \ + mock.patch('os.geteuid', return_value=0, create=True), \ + mock.patch('subprocess.call', return_value=0) as subprocess_call: + _install_deps_for_psycopg2() + + azure_linux_packages = ['gcc', 'libpq-devel', 'python3-devel', 'binutils', 'glibc-devel', 'kernel-headers'] + subprocess_call.assert_any_call(['tdnf', 'install', '-y'] + azure_linux_packages) + def _setup_cmd(self): cmd = mock.MagicMock() cmd.cli_ctx = DummyCli()