From 99731afbb5d2b01488bb0480c7e6e662597e041c Mon Sep 17 00:00:00 2001 From: Hang Date: Thu, 7 Jul 2022 14:59:04 +0800 Subject: [PATCH] [Packaging] Build RPM for Fedora (#22945) * Merge dev * Apply suggestions from code review Co-authored-by: Jiashuo Li <4003950+jiasli@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Jiashuo Li <4003950+jiasli@users.noreply.github.com> * Minor fix Co-authored-by: Jiashuo Li <4003950+jiasli@users.noreply.github.com> --- azure-pipelines.yml | 59 ++++++++++++++++++----- scripts/ci/check_pull_request.py | 2 +- scripts/release/rpm/azure-cli.spec | 6 +++ scripts/release/rpm/fedora.dockerfile | 7 ++- scripts/release/rpm/pipeline.sh | 6 ++- scripts/release/rpm/test_rpm_in_docker.sh | 12 +++-- scripts/release/rpm/test_rpm_package.py | 5 +- 7 files changed, 72 insertions(+), 25 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9bef01b1f73..6b108c8b5d2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -627,14 +627,27 @@ jobs: TargetPath: $(Build.ArtifactStagingDirectory) ArtifactName: rpm-mariner$(tag) - # TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. -- job: BuildRpmPackageUbi8 - displayName: Build Rpm Package Red Hat Universal Base Image 8 +- job: BuildRpmPackages + displayName: Build Rpm Packages # Do not run this job for Pull Requests due to the slowness condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: vmImage: 'ubuntu-20.04' + strategy: + matrix: + Red Hat Universal Base Image 8: + image: ubi8 + tag: 8.4 + artifact: rpm-ubi8 + Fedora35: + image: fedora + tag: 35 + artifact: rpm-fedora35 + Fedora36: + image: fedora + tag: 36 + artifact: rpm-fedora36 steps: - task: Bash@3 displayName: 'Build Rpm Package' @@ -642,7 +655,8 @@ jobs: targetType: 'filePath' filePath: scripts/release/rpm/pipeline.sh env: - IMAGE: ubi8 + IMAGE: $(image) + TAG: $(tag) - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: 'SBOM' inputs: @@ -651,16 +665,39 @@ jobs: displayName: 'Publish Artifact: rpm' inputs: TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: rpm-ubi8 + ArtifactName: $(artifact) - job: TestRpmPackage displayName: Test Rpm Package timeoutInMinutes: 120 - dependsOn: BuildRpmPackageUbi8 + dependsOn: BuildRpmPackages condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: vmImage: 'ubuntu-20.04' + strategy: + matrix: + Red Hat Universal Base Image 8: + artifact: rpm-ubi8 + distro: el8 + image: registry.access.redhat.com/ubi8/ubi:8.4 + python_package: python39 + python_cmd: python3.9 + pip_cmd: pip3.9 + Fedora35: + artifact: rpm-fedora35 + distro: fc35 + image: fedora:35 + python_package: python3 + python_cmd: python3 + pip_cmd: pip3 + Fedora36: + artifact: rpm-fedora36 + distro: fc36 + image: fedora:36 + python_package: python3 + python_cmd: python3 + pip_cmd: pip3 steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Metadata' @@ -673,21 +710,19 @@ jobs: displayName: 'Download Build Artifacts' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: rpm-ubi8 + artifactName: $(artifact) - bash: | set -ex CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=azure-cli-$CLI_VERSION-1.el8.x86_64.rpm + RPM_NAME=azure-cli-$CLI_VERSION-1.${DISTRO}.x86_64.rpm RPM_FILE=$SYSTEM_ARTIFACTSDIRECTORY/rpm/$RPM_NAME - echo "== Test rpm package on Red Hat UBI 8 ==" - - IMAGE=registry.access.redhat.com/ubi8/ubi:8.4 + echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" + docker run --rm -e RPM_NAME=$RPM_NAME -e PYTHON_PACKAGE=${PYTHON_PACKAGE} -e PYTHON_CMD=${PYTHON_CMD} -e PIP_CMD=${PIP_CMD} -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" displayName: 'Test Rpm Package' diff --git a/scripts/ci/check_pull_request.py b/scripts/ci/check_pull_request.py index c76d2b62f4d..69c629936b8 100644 --- a/scripts/ci/check_pull_request.py +++ b/scripts/ci/check_pull_request.py @@ -121,7 +121,7 @@ def check_line(line): logger.error(' ' * idx + '↑') error_flag = True # az xxx commands must be enclosed in `, e.g., `az vm` - if i == 'a' and line[idx + 1] == 'z' and line[idx + 2] == ' ': + if idx + 2 < len(line) and i == 'a' and line[idx + 1] == 'z' and line[idx + 2] == ' ': command = 'az ' index = idx + 3 while index < len(line) and line[index] != ':': diff --git a/scripts/release/rpm/azure-cli.spec b/scripts/release/rpm/azure-cli.spec index 1adc7df502b..040618f1cf2 100644 --- a/scripts/release/rpm/azure-cli.spec +++ b/scripts/release/rpm/azure-cli.spec @@ -37,6 +37,12 @@ BuildRequires: %{python_package}-devel %global _python_bytecompile_errors_terminate_build 0 +# To get rid of `cannot open linker script file` error on Fedora36. If %_package_note_file is undefined, the +# linker script will not be generated. Related bug: https://bugzilla.redhat.com/show_bug.cgi?id=2043092 +# Ref: https://src.fedoraproject.org/rpms/ruby/c/a0bcb33eaa666d3e1d08ca45e77161ca05611487?branch=rawhide +# https://src.fedoraproject.org/rpms/redhat-rpm-config//blob/rawhide/f/buildflags.md +%undefine _package_note_file + %description A great cloud needs great tools; we're excited to introduce Azure CLI, our next generation multi-platform command line experience for Azure. diff --git a/scripts/release/rpm/fedora.dockerfile b/scripts/release/rpm/fedora.dockerfile index 8b92bb8922f..531216f0d2e 100644 --- a/scripts/release/rpm/fedora.dockerfile +++ b/scripts/release/rpm/fedora.dockerfile @@ -4,20 +4,19 @@ FROM fedora:${tag} AS build-env ARG cli_version=dev RUN dnf update -y -RUN dnf install -y wget rpm-build gcc libffi-devel python3-devel python3-virtualenv openssl-devel make bash coreutils diffutils patch dos2unix perl +RUN dnf install -y wget rpm-build gcc libffi-devel python3-devel openssl-devel make bash coreutils diffutils patch dos2unix perl WORKDIR /azure-cli COPY . . RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ - REPO_PATH=$(pwd) CLI_VERSION=$cli_version rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ + REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=python3 PYTHON_CMD=python3 \ + rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ cp /root/rpmbuild/RPMS/x86_64/azure-cli-${cli_version}-1.*.x86_64.rpm /azure-cli-dev.rpm FROM fedora:${tag} AS execution-env -RUN dnf install -y python3 python3-virtualenv - COPY --from=build-env /azure-cli-dev.rpm ./ RUN rpm -i ./azure-cli-dev.rpm && \ az --version diff --git a/scripts/release/rpm/pipeline.sh b/scripts/release/rpm/pipeline.sh index f6f2d5384a1..d1d6ec84331 100755 --- a/scripts/release/rpm/pipeline.sh +++ b/scripts/release/rpm/pipeline.sh @@ -6,8 +6,10 @@ set -exv : "${BUILD_STAGINGDIRECTORY:?BUILD_STAGINGDIRECTORY environment variable not set.}" -# IMAGE should be one of 'centos7', 'ubi8' +# IMAGE should be one of 'centos7', 'ubi8' or 'fedora' : "${IMAGE:?IMAGE environment variable not set.}" +# TAG should be 'centos7', '8.4' or Fedora version number +: "${TAG:?TAG environment variable not set.}" CLI_VERSION=`cat src/azure-cli/azure/cli/__main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` @@ -15,6 +17,7 @@ CLI_VERSION=`cat src/azure-cli/azure/cli/__main__.py | grep __version__ | sed s/ docker build \ --target build-env \ --build-arg cli_version=${CLI_VERSION} \ + --build-arg tag=${TAG} \ -f ./scripts/release/rpm/${IMAGE}.dockerfile \ -t azure/azure-cli:${IMAGE}-builder \ . @@ -22,6 +25,7 @@ docker build \ # Continue the previous build, and create a container that has the current azure-cli build but not the source code. docker build \ --build-arg cli_version=${CLI_VERSION} \ + --build-arg tag=${TAG} \ -f ./scripts/release/rpm/${IMAGE}.dockerfile \ -t azure/azure-cli:${IMAGE} \ . diff --git a/scripts/release/rpm/test_rpm_in_docker.sh b/scripts/release/rpm/test_rpm_in_docker.sh index cf806a49c3d..45e9a2e07cc 100644 --- a/scripts/release/rpm/test_rpm_in_docker.sh +++ b/scripts/release/rpm/test_rpm_in_docker.sh @@ -5,13 +5,9 @@ set -exv export USERNAME=azureuser -PYTHON_PACKAGE=python39 -PYTHON_CMD=python3.9 -PIP_CMD=pip3.9 - dnf --nogpgcheck install /mnt/rpm/$RPM_NAME -y -dnf install git gcc $PYTHON_PACKAGE-devel -y +dnf install git gcc $PYTHON_PACKAGE-devel findutils -y ln -s -f /usr/bin/$PYTHON_CMD /usr/bin/python ln -s -f /usr/bin/$PIP_CMD /usr/bin/pip @@ -21,6 +17,12 @@ time az --version cd /azure-cli/ pip install wheel ./scripts/ci/build.sh + +# From Fedora36, when using `pip install --prefix` with root privileges, the package is installed into `{prefix}/local/lib`. +# In order to keep the original installation path, I have to set RPM_BUILD_ROOT +# Ref https://docs.fedoraproject.org/en-US/fedora/latest/release-notes/developers/Development_Python/#_pipsetup_py_installation_with_prefix +export RPM_BUILD_ROOT=/ + pip install pytest --prefix /usr/lib64/az pip install pytest-xdist --prefix /usr/lib64/az diff --git a/scripts/release/rpm/test_rpm_package.py b/scripts/release/rpm/test_rpm_package.py index 95845cae4d1..f3ee8ffb0de 100644 --- a/scripts/release/rpm/test_rpm_package.py +++ b/scripts/release/rpm/test_rpm_package.py @@ -7,10 +7,11 @@ import sys import subprocess -root_dir = '/usr/lib64/az/lib/python3.9/site-packages/azure/cli/command_modules' +python_version = os.listdir('/usr/lib64/az/lib/')[0] +root_dir = f'/usr/lib64/az/lib/{python_version}/site-packages/azure/cli/command_modules' mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__'] -pytest_base_cmd = 'PYTHONPATH=/usr/lib64/az/lib/python3.9/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN' +pytest_base_cmd = f'PYTHONPATH=/usr/lib64/az/lib/{python_version}/site-packages python -m pytest -x -v --boxed -p no:warnings --log-level=WARN' pytest_parallel_cmd = '{} -n auto'.format(pytest_base_cmd) serial_test_modules = ['botservice', 'network', 'cloud', 'appservice']