[Packaging] Build RPM for Fedora (#22945)

* Merge dev * Apply suggestions from code review Co-authored-by: Jiashuo Li <[email protected]> * Apply suggestions from code review Co-authored-by: Jiashuo Li <[email protected]> * Minor fix Co-authored-by: Jiashuo Li <[email protected]>
Azure · Jul 7, 2022 · 99731af · 99731af
1 parent 5abef23
commit 99731af
Show file tree

Hide file tree

Showing 7 changed files with 72 additions and 25 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -627,22 +627,36 @@ jobs:
       TargetPath: $(Build.ArtifactStagingDirectory)
       ArtifactName: rpm-mariner$(tag)
 
-
 # TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate.
-- job: BuildRpmPackageUbi8
-  displayName: Build Rpm Package Red Hat Universal Base Image 8
+- job: BuildRpmPackages
+  displayName: Build Rpm Packages
   # Do not run this job for Pull Requests due to the slowness
   condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule'))
   pool:
     vmImage: 'ubuntu-20.04'
+  strategy:
+    matrix:
+      Red Hat Universal Base Image 8:
+        image: ubi8
+        tag: 8.4
+        artifact: rpm-ubi8
+      Fedora35:
+        image: fedora
+        tag: 35
+        artifact: rpm-fedora35
+      Fedora36:
+        image: fedora
+        tag: 36
+        artifact: rpm-fedora36
   steps:
   - task: Bash@3
     displayName: 'Build Rpm Package'
     inputs:
       targetType: 'filePath'
       filePath: scripts/release/rpm/pipeline.sh
     env:
-      IMAGE: ubi8
+      IMAGE: $(image)
+      TAG: $(tag)
   - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0
     displayName: 'SBOM'
     inputs:
@@ -651,16 +665,39 @@ jobs:
     displayName: 'Publish Artifact: rpm'
     inputs:
       TargetPath: $(Build.ArtifactStagingDirectory)
-      ArtifactName: rpm-ubi8
+      ArtifactName: $(artifact)
 
 
 - job: TestRpmPackage
   displayName: Test Rpm Package
   timeoutInMinutes: 120
-  dependsOn: BuildRpmPackageUbi8
+  dependsOn: BuildRpmPackages
   condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule'))
   pool:
     vmImage: 'ubuntu-20.04'
+  strategy:
+    matrix:
+      Red Hat Universal Base Image 8:
+        artifact: rpm-ubi8
+        distro: el8
+        image: registry.access.redhat.com/ubi8/ubi:8.4
+        python_package: python39
+        python_cmd: python3.9
+        pip_cmd: pip3.9
+      Fedora35:
+        artifact: rpm-fedora35
+        distro: fc35
+        image: fedora:35
+        python_package: python3
+        python_cmd: python3
+        pip_cmd: pip3
+      Fedora36:
+        artifact: rpm-fedora36
+        distro: fc36
+        image: fedora:36
+        python_package: python3
+        python_cmd: python3
+        pip_cmd: pip3
   steps:
   - task: DownloadPipelineArtifact@1
     displayName: 'Download Metadata'
@@ -673,21 +710,19 @@ jobs:
     displayName: 'Download Build Artifacts'
     inputs:
       TargetPath: '$(Build.ArtifactStagingDirectory)/rpm'
-      artifactName: rpm-ubi8
+      artifactName: $(artifact)
 
 
   - bash: |
       set -ex
       
       CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version`
-      RPM_NAME=azure-cli-$CLI_VERSION-1.el8.x86_64.rpm
+      RPM_NAME=azure-cli-$CLI_VERSION-1.${DISTRO}.x86_64.rpm
       RPM_FILE=$SYSTEM_ARTIFACTSDIRECTORY/rpm/$RPM_NAME
       
-      echo "== Test rpm package on Red Hat UBI 8 =="
-      
-      IMAGE=registry.access.redhat.com/ubi8/ubi:8.4
+      echo "== Test rpm package on ${IMAGE} =="
       docker pull $IMAGE
-      docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh"
+      docker run --rm -e RPM_NAME=$RPM_NAME -e PYTHON_PACKAGE=${PYTHON_PACKAGE} -e PYTHON_CMD=${PYTHON_CMD} -e PIP_CMD=${PIP_CMD} -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh"
 
     displayName: 'Test Rpm Package'
 

diff --git a/scripts/ci/check_pull_request.py b/scripts/ci/check_pull_request.py
@@ -121,7 +121,7 @@ def check_line(line):
                 logger.error(' ' * idx + '↑')
                 error_flag = True
         # az xxx commands must be enclosed in `, e.g., `az vm`
-        if i == 'a' and line[idx + 1] == 'z' and line[idx + 2] == ' ':
+        if idx + 2 < len(line) and i == 'a' and line[idx + 1] == 'z' and line[idx + 2] == ' ':
             command = 'az '
             index = idx + 3
             while index < len(line) and line[index] != ':':

diff --git a/scripts/release/rpm/azure-cli.spec b/scripts/release/rpm/azure-cli.spec
@@ -37,6 +37,12 @@ BuildRequires:  %{python_package}-devel
 
 %global _python_bytecompile_errors_terminate_build 0
 
+# To get rid of `cannot open linker script file` error on Fedora36. If %_package_note_file is undefined, the
+# linker script will not be generated. Related bug: https://bugzilla.redhat.com/show_bug.cgi?id=2043092
+# Ref: https://src.fedoraproject.org/rpms/ruby/c/a0bcb33eaa666d3e1d08ca45e77161ca05611487?branch=rawhide
+#      https://src.fedoraproject.org/rpms/redhat-rpm-config//blob/rawhide/f/buildflags.md
+%undefine _package_note_file
+
 %description
 A great cloud needs great tools; we're excited to introduce Azure CLI,
  our next generation multi-platform command line experience for Azure.

diff --git a/scripts/release/rpm/fedora.dockerfile b/scripts/release/rpm/fedora.dockerfile
@@ -4,20 +4,19 @@ FROM fedora:${tag} AS build-env
 ARG cli_version=dev
 
 RUN dnf update -y
-RUN dnf install -y wget rpm-build gcc libffi-devel python3-devel python3-virtualenv openssl-devel make bash coreutils diffutils patch dos2unix perl
+RUN dnf install -y wget rpm-build gcc libffi-devel python3-devel openssl-devel make bash coreutils diffutils patch dos2unix perl
 
 WORKDIR /azure-cli
 
 COPY . .
 
 RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \
-    REPO_PATH=$(pwd) CLI_VERSION=$cli_version rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \
+    REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=python3 PYTHON_CMD=python3 \
+    rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \
     cp /root/rpmbuild/RPMS/x86_64/azure-cli-${cli_version}-1.*.x86_64.rpm /azure-cli-dev.rpm
 
 FROM fedora:${tag} AS execution-env
 
-RUN dnf install -y python3 python3-virtualenv
-
 COPY --from=build-env /azure-cli-dev.rpm ./
 RUN rpm -i ./azure-cli-dev.rpm && \
     az --version
diff --git a/scripts/release/rpm/pipeline.sh b/scripts/release/rpm/pipeline.sh
@@ -6,22 +6,26 @@ set -exv
 
 : "${BUILD_STAGINGDIRECTORY:?BUILD_STAGINGDIRECTORY environment variable not set.}"
 
-# IMAGE should be one of 'centos7', 'ubi8'
+# IMAGE should be one of 'centos7', 'ubi8' or 'fedora'
 : "${IMAGE:?IMAGE environment variable not set.}"
+# TAG should be 'centos7', '8.4' or Fedora version number
+: "${TAG:?TAG environment variable not set.}"
 
 CLI_VERSION=`cat src/azure-cli/azure/cli/__main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// |  sed s/\"//g`
 
 # Create a container image that includes the source code and a built RPM using this file.
 docker build \
     --target build-env \
     --build-arg cli_version=${CLI_VERSION} \
+    --build-arg tag=${TAG} \
     -f ./scripts/release/rpm/${IMAGE}.dockerfile \
     -t azure/azure-cli:${IMAGE}-builder \
     .
 
 # Continue the previous build, and create a container that has the current azure-cli build but not the source code.
 docker build \
     --build-arg cli_version=${CLI_VERSION} \
+    --build-arg tag=${TAG} \
     -f ./scripts/release/rpm/${IMAGE}.dockerfile \
     -t azure/azure-cli:${IMAGE} \
     .

diff --git a/scripts/release/rpm/test_rpm_in_docker.sh b/scripts/release/rpm/test_rpm_in_docker.sh
@@ -5,13 +5,9 @@ set -exv
 
 export USERNAME=azureuser
 
-PYTHON_PACKAGE=python39
-PYTHON_CMD=python3.9
-PIP_CMD=pip3.9
-
 dnf --nogpgcheck install /mnt/rpm/$RPM_NAME -y
 
-dnf install git gcc $PYTHON_PACKAGE-devel -y
+dnf install git gcc $PYTHON_PACKAGE-devel findutils -y
 
 ln -s -f /usr/bin/$PYTHON_CMD /usr/bin/python
 ln -s -f /usr/bin/$PIP_CMD /usr/bin/pip
@@ -21,6 +17,12 @@ time az --version
 cd /azure-cli/
 pip install wheel
 ./scripts/ci/build.sh
+
+# From Fedora36, when using `pip install --prefix` with root privileges, the package is installed into `{prefix}/local/lib`.
+# In order to keep the original installation path, I have to set RPM_BUILD_ROOT
+# Ref https://docs.fedoraproject.org/en-US/fedora/latest/release-notes/developers/Development_Python/#_pipsetup_py_installation_with_prefix
+export RPM_BUILD_ROOT=/
+
 pip install pytest --prefix /usr/lib64/az
 pip install pytest-xdist --prefix /usr/lib64/az
 

diff --git a/scripts/release/rpm/test_rpm_package.py b/scripts/release/rpm/test_rpm_package.py
@@ -7,10 +7,11 @@
 import sys
 import subprocess
 
-root_dir = '/usr/lib64/az/lib/python3.9/site-packages/azure/cli/command_modules'
+python_version = os.listdir('/usr/lib64/az/lib/')[0]
+root_dir = f'/usr/lib64/az/lib/{python_version}/site-packages/azure/cli/command_modules'
 mod_list = [mod for mod in sorted(os.listdir(root_dir)) if os.path.isdir(os.path.join(root_dir, mod)) and mod != '__pycache__']
 
-pytest_base_cmd = 'PYTHONPATH=/usr/lib64/az/lib/python3.9/site-packages python3 -m pytest -x -v --boxed -p no:warnings --log-level=WARN'
+pytest_base_cmd = f'PYTHONPATH=/usr/lib64/az/lib/{python_version}/site-packages python -m pytest -x -v --boxed -p no:warnings --log-level=WARN'
 pytest_parallel_cmd = '{} -n auto'.format(pytest_base_cmd)
 serial_test_modules = ['botservice', 'network', 'cloud', 'appservice']