DAOS-15079 test: VMD Hot Plug - Support rpc.py in spdk-tools package (#…

…61)

Update packaging/.

We need to support rpc.py from spdk-tools package (RPM) so that
we can use the script in CI.

Make the changes in spdk.spec so that rpc.py is included in the
RPM. After installing the RPM, we should be able to use the
script by calling:
/usr/share/spdk/scripts/rpc.py

Skip CentOS7 build.

Fix rpmlint error.

Add examples installation for Debian to spdk-tools.

Install all scripts from spdk-tools.

Skip-list-master: test_daos_rebuild_ec:DAOS-14982 test_daos_degraded_ec:DAOS-14982 test_ms_failover:DAOS-16103 test_cart_rpc:DAOS-15989
Skip-list-release/2.6: test_dfuse_daos_build_wb:DAOS-16168 test_osa_offline_reintegration_without_checksum:DAOS-15608 test_daos_extend_simple:DAOS-15640

Signed-off-by: Tomasz Gromadzki <[email protected]>
Signed-off-by: Makito Kano <[email protected]>
Signed-off-by: Gromadzki, Tomasz <[email protected]>
Signed-off-by: Jan Michalski <[email protected]>
Co-authored-by: Tomasz Gromadzki <[email protected]>
Co-authored-by: Jan Michalski <[email protected]>

Jenkinsfile

@@ -41,5 +41,5 @@
 //@Library(value="pipeline-lib@your_branch") _
 
 /* groovylint-disable-next-line CompileStatic */
-packageBuildingPipelineDAOSTest(['distros': ['centos7', 'el8', 'el9', 'leap15', 'ubuntu20.04'],
+packageBuildingPipelineDAOSTest(['distros': ['el8', 'el9', 'leap15', 'ubuntu20.04'],
                                 'test-tag': 'pr'])

debian/changelog

@@ -1,3 +1,11 @@
+spdk (22.01.2-6) unstable; urgency=medium
+
+  [ Tomasz Gromadzki ]
+  * Add rpc.py to spdk-tools package.
+  * Synchronize Debian package with RPMs by adding examples to spdk-tools package.
+
+ --  Tomasz Gromadzki <[email protected]>  Tue, 23 Apr 2024 00:00:00 +0000
+
 spdk (22.01.2-3) unstable; urgency=medium
 
   [ Tom Nabarro ]

debian/control

@@ -3,7 +3,7 @@ Priority: optional
 Maintainer: daos-stack <[email protected]>
 Build-Depends:
  debhelper,
- python,
+ python3,
  libdpdk-dev (>= 21.11.2),
  librdmacm-dev,
  libibverbs-dev,

debian/libspdk-dev.install

@@ -1,4 +1,4 @@
 usr/include/*
 lib/x86_64-linux-gnu/lib*.so
 lib/x86_64-linux-gnu/lib*.a
-lib/x86_64-linux-gnu/pkgconfig/*
+lib/x86_64-linux-gnu/pkgconfig/*

debian/libspdk3.install

@@ -1 +1 @@
-lib/x86_64-linux-gnu/lib*.so.*
+lib/x86_64-linux-gnu/lib*.so.*

debian/rules

@@ -6,6 +6,13 @@ export DEB_BUILD_MAINT_OPTIONS = hardening=+all
 # https://github.com/spdk/spdk/issues/2011
 export DEB_CFLAGS_MAINT_APPEND = $(shell pkg-config --cflags libdpdk)
 
+prefix := /usr
+
+cflags := -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 \
+	-Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong \
+	-grecord-gcc-switches -m64 -mtune=generic -fasynchronous-unwind-tables \
+	-fstack-clash-protection -fcf-protection
+
 %:
 	dh $@
 
@@ -41,11 +48,17 @@ override_dh_auto_configure:
 	            --without-vtune           \
 	            --with-shared
 
-#override_dh_auto_build:
-#	make V=1 # -j 8
+override_dh_auto_build:
+	export CFLAGS='$(cflags)'
+	export CXXFLAGS='$(cflags)'
+	export FFLAGS='$(cflags)'
+	export LDFLAGS='-Wl,-z,relro -Wl,-z,now'
+	dh_auto_build -- CONFIG_ARCH=haswell V=1
 
 override_dh_auto_install:
-	dh_auto_install -- prefix=/usr libdir=/lib/x86_64-linux-gnu/ datadir=/usr/share
+	mv build/examples/lsvmd build/examples/spdk_lsvmd
+	mv build/examples/nvme_manage build/examples/spdk_nvme_manage
+	dh_auto_install -- prefix=$(prefix) libdir=/lib/x86_64-linux-gnu/ datadir=$(prefix)/share
 
 override_dh_missing:
 	dh_missing --fail-missing

debian/spdk-tools.dirs

@@ -0,0 +1,4 @@
+usr/share/spdk-tools/scripts
+usr/include/spdk
+usr/share/spdk-tools/python
+usr/share/spdk-tools/python/rpc

debian/spdk-tools.install

@@ -1 +1,7 @@
-usr/bin/*
+usr/bin/*
+scripts/{setup,common}.sh usr/share/spdk-tools/scripts
+include/spdk/pci_ids.h usr/include/spdk
+scripts/rpc.py usr/share/spdk-tools/python
+scripts/rpc/*.py usr/share/spdk-tools/python/rpc
+build/examples/spdk_lsvmd usr/bin/
+build/examples/spdk_nvme_manage usr/bin/

packaging/Dockerfile.mockbuild

@@ -1,11 +1,11 @@
 #
-# Copyright 2018-2023 Intel Corporation
+# Copyright 2018-2024 Intel Corporation
 #
 # 'recipe' for Docker to build an RPM
 #
 
 # Pull base image
-ARG FVERSION=38
+ARG FVERSION=latest
 FROM fedora:$FVERSION
 # Needed for later use of FVERSION
 ARG FVERSION
@@ -48,16 +48,24 @@ RUN dnf -y upgrade && \
 # https://github.com/rpm-software-management/rpmlint/pull/795 in it
 # But make sure to patch after dnf upgrade so that an upgraded rpmlint
 # RPM doesn't wipe out our patch
+# Ditto for the patch to zero and display ccache stats
+# https://github.com/rpm-software-management/mock/pull/1299
 ARG PACKAGINGDIR=packaging
-COPY ${PACKAGINGDIR}/rpmlint--ignore-unused-rpmlintrc.patch .
+COPY ${PACKAGINGDIR}/*.patch ./
 RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') &&                      \
      if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then                               \
-         if ! patch -p1; then                                                                  \
+         if ! patch -p1 < $OLDPWD/rpmlint--ignore-unused-rpmlintrc.patch; then                 \
              exit 1;                                                                           \
          fi;                                                                                   \
          rm -f rpmlint/__pycache__/{cli,lint}.*.pyc;                                           \
-     fi) < rpmlint--ignore-unused-rpmlintrc.patch;                                             \
-    rm -f  rpmlint--ignore-unused-rpmlintrc.patch
+     fi;                                                                                       \
+     if ! grep _ccachePostBuildHook mockbuild/plugins/ccache.py; then                          \
+         if ! patch -p3 < $OLDPWD/ccache-stats.patch; then                                     \
+             exit 1;                                                                           \
+         fi;                                                                                   \
+         rm -f mockbuild/plugins/__pycache__/ccache.*.pyc;                                     \
+     fi);                                                                                      \
+     rm -f rpmlint--ignore-unused-rpmlintrc.patch ccache-stats.patch
 
 # show the release that was built
 ARG CACHEBUST

packaging/Dockerfile.ubuntu.20.04

@@ -18,26 +18,23 @@ RUN if [ -n "$REPO_FILE_URL" ]; then                                 \
         true > ../sources.list &&                                    \
         mv daos_ci-ubuntu20.04-artifactory.list.tmp                  \
            daos_ci-ubuntu20.04-artifactory.list;                     \
+        url="${REPO_FILE_URL%/*/}/hpe-ilorest-ubuntu-bionic-proxy/"; \
+    else                                                             \
+        url="https://downloads.linux.hpe.com/SDR/repo/ilorest/";     \
     fi;                                                              \
     cd -;                                                            \
-    curl -f -O "$REPO_FILE_URL"esad_repo.key;                        \
-    gpg --no-default-keyring --keyring ./temp-keyring.gpg            \
-        --import esad_repo.key;                                      \
     mkdir -p /usr/local/share/keyrings/;                             \
+    curl -f -O "$url"GPG-KEY-hprest;                                 \
+    gpg --no-default-keyring --keyring ./temp-keyring.gpg            \
+        --import GPG-KEY-hprest;                                     \
     gpg --no-default-keyring --keyring ./temp-keyring.gpg --export   \
-        --output /usr/local/share/keyrings/daos-stack-public.gpg;    \
+        --output /usr/local/share/keyrings/hpe-sdr-public.gpg;       \
     rm ./temp-keyring.gpg;                                           \
-    url_prefix=https://downloads.linux.hpe.com/SDR/;                 \
-    for url in hpPublicKey2048.pub                                   \
-               hpPublicKey2048_key1.pub                              \
-               hpePublicKey2048_key1.pub; do                         \
-        curl -f -O "$url_prefix$url";                                \
-        gpg --no-default-keyring --keyring ./temp-keyring.gpg        \
-            --import "$(basename $url)";                             \
-    done;                                                            \
+    curl -f -O "$REPO_FILE_URL"esad_repo.key;                        \
+    gpg --no-default-keyring --keyring ./temp-keyring.gpg            \
+        --import esad_repo.key;                                      \
     gpg --no-default-keyring --keyring ./temp-keyring.gpg --export   \
-        --output /usr/local/share/keyrings/hpe-sdr-public.gpg;       \
-    rm ./temp-keyring.gpg
+        --output /usr/local/share/keyrings/daos-stack-public.gpg
 
 # Install basic tools
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \

packaging/Makefile_distro_vars.mk

@@ -83,6 +83,14 @@ DISTRO_VERSION  ?= $(VERSION_ID)
 ORIG_TARGET_VER := 15.4
 SED_EXPR        := 1p
 endif
+ifeq ($(CHROOT_NAME),opensuse-leap-15.5-x86_64)
+VERSION_ID      := 15.5
+DISTRO_ID       := sl15.5
+DISTRO_BASE     := LEAP_15
+DISTRO_VERSION  ?= $(VERSION_ID)
+ORIG_TARGET_VER := 15.5
+SED_EXPR        := 1p
+endif
 endif
 ifeq ($(ID),centos)
 ID = el

packaging/Makefile_packaging.mk

@@ -35,7 +35,11 @@ TEST_PACKAGES ?= ${NAME}
 # unfortunately we cannot always name the repo the same as the project
 REPO_NAME ?= $(NAME)
 
+ifneq ($(CI_PR_REPOS),)
+PR_REPOS                 ?= $(CI_PR_REPOS)
+else
 PR_REPOS                 ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos: *\(.*\)/\1/p')
+endif
 LEAP_15_PR_REPOS         ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-leap15: *\(.*\)/\1/p')
 EL_7_PR_REPOS            ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el7: *\(.*\)/\1/p')
 EL_8_PR_REPOS            ?= $(shell git show -s --format=%B | sed -ne 's/^PR-repos-el8: *\(.*\)/\1/p')
@@ -54,7 +58,8 @@ RPM_BUILD_OPTIONS := $(BUILD_DEFINES)
 GIT_DIFF_EXCLUDES := $(PATCH_EXCLUDE_FILES:%=':!%')
 endif
 
-COMMON_RPM_ARGS   = --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES)
+FVERSION         ?= latest
+COMMON_RPM_ARGS  := --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES)
 SPEC             := $(shell if [ -f $(NAME)-$(DISTRO_BASE).spec ]; then echo $(NAME)-$(DISTRO_BASE).spec; else echo $(NAME).spec; fi)
 VERSION           = $(eval VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p'))$(VERSION)
 DEB_RVERS        := $(subst $(DOT),\$(DOT),$(VERSION))
@@ -363,12 +368,14 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE)
 	LOCAL_REPOS='$(LOCAL_REPOS)'                            \
 	ARTIFACTORY_URL="$(ARTIFACTORY_URL)"                    \
 	DISTRO_VERSION="$(DISTRO_VERSION)"                      \
+	PACKAGE="$(NAME)"                                       \
 	TARGET="$<"                                             \
 	packaging/rpm_chrootbuild
 endif
 
 podman_chrootbuild:
 	if ! podman build --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \
+	                  --build-arg FVERSION=$(FVERSION)           \
 	                  -t $(subst +,-,$(CHROOT_NAME))-chrootbuild \
 	                  -f packaging/Dockerfile.mockbuild .; then  \
 		echo "Container build failed";                           \
@@ -386,7 +393,9 @@ podman_chrootbuild:
 	                                 exit 1;                                                                                        \
 	                             fi;                                                                                                \
 	                             rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm |                                          \
-	                                 grep -v -e debuginfo -e debugsource -e src.rpm)'
+	                                 grep -v -e debuginfo -e debugsource -e src.rpm)'; then                                         \
+		exit 1;                                                                                                                 \
+	fi
 
 docker_chrootbuild:
 	if ! $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild   \
@@ -419,6 +428,8 @@ packaging_check:
 	          --exclude libfabric.spec                      \
 	          --exclude Makefile                            \
 	          --exclude README.md                           \
+	          --exclude SECURITY.md                         \
+	          --exclude LICENSE                             \
 	          --exclude _topdir                             \
 	          --exclude \*.tar.\*                           \
 	          --exclude \*.code-workspace                   \

packaging/ccache-stats.patch

@@ -0,0 +1,66 @@
+From e87d916d7f49ea4949973adf0f09e9e5bf891e03 Mon Sep 17 00:00:00 2001
+From: "Brian J. Murrell" <[email protected]>
+Date: Tue, 30 Jan 2024 11:03:12 -0500
+Subject: [PATCH 1/2] Show ccache stats at the end of the build
+
+Zero the ccache stats at the beginning of the build and then display the
+ccache stats at the end of the build to see how effective ccache was.
+
+Signed-off-by: Brian J. Murrell <[email protected]>
+---
+ mock/py/mockbuild/plugins/ccache.py | 11 +++++++++++
+ 1 file changed, 11 insertions(+)
+
+diff --git a/mock/py/mockbuild/plugins/ccache.py b/mock/py/mockbuild/plugins/ccache.py
+index 2666ad9fc..1080ffe68 100644
+--- a/mock/py/mockbuild/plugins/ccache.py
++++ b/mock/py/mockbuild/plugins/ccache.py
+@@ -35,6 +35,7 @@ def __init__(self, plugins, conf, buildroot):
+         buildroot.preexisting_deps.append("ccache")
+         plugins.add_hook("prebuild", self._ccacheBuildHook)
+         plugins.add_hook("preinit", self._ccachePreInitHook)
++        plugins.add_hook("postbuild", self._ccachePostBuildHook)
+         buildroot.mounts.add(
+             BindMountPoint(srcpath=self.ccachePath, bindpath=buildroot.make_chroot_path("/var/tmp/ccache")))
+
+@@ -47,6 +48,9 @@ def __init__(self, plugins, conf, buildroot):
+     @traceLog()
+     def _ccacheBuildHook(self):
+         self.buildroot.doChroot(["ccache", "-M", str(self.ccache_opts['max_cache_size'])], shell=False)
++        # zero ccache stats
++        getLog().info("Zero ccache stats:")
++        self.buildroot.doChroot(["ccache", "--zero-stats"], printOutput=True, shell=False)
+
+     # set up the ccache dir.
+     # we also set a few variables used by ccache to find the shared cache.
+@@ -61,3 +65,10 @@ def _ccachePreInitHook(self):
+         file_util.mkdirIfAbsent(self.buildroot.make_chroot_path('/var/tmp/ccache'))
+         file_util.mkdirIfAbsent(self.ccachePath)
+         self.buildroot.uid_manager.changeOwner(self.ccachePath, recursive=True)
++
++    # get some cache stats
++    def _ccachePostBuildHook(self):
++        # show the cache hit stats
++        getLog().info("ccache stats:")
++        self.buildroot.doChroot(["ccache", "--show-stats"], printOutput=True, shell=False)
+++
+
+From bfd3a7e1bb47d28ee60a94cb5985c1f66476475f Mon Sep 17 00:00:00 2001
+From: "Brian J. Murrell" <[email protected]>
+Date: Tue, 30 Jan 2024 11:17:48 -0500
+Subject: [PATCH 2/2] Remove extraneous line
+
+Signed-off-by: Brian J. Murrell <[email protected]>
+---
+ mock/py/mockbuild/plugins/ccache.py | 1 -
+ 1 file changed, 1 deletion(-)
+
+diff --git a/mock/py/mockbuild/plugins/ccache.py b/mock/py/mockbuild/plugins/ccache.py
+index 1080ffe68..1a20846d3 100644
+--- a/mock/py/mockbuild/plugins/ccache.py
++++ b/mock/py/mockbuild/plugins/ccache.py
+@@ -71,4 +71,3 @@ def _ccachePostBuildHook(self):
+         # show the cache hit stats
+         getLog().info("ccache stats:")
+         self.buildroot.doChroot(["ccache", "--show-stats"], printOutput=True, shell=False)
+-+

packaging/get_base_branch

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# find the base branch of the current branch
+
+set -eux -o pipefail
+IFS=' ' read -r -a add_bases <<< "${1:-}"
+origin=origin
+mapfile -t all_bases < <(echo "master"
+                         git branch -r | sed -ne "/^  $origin\\/release\\/[0-9]/s/^  $origin\\///p")
+all_bases+=("${add_bases[@]}")
+TARGET="master"
+min_diff=-1
+for base in "${all_bases[@]}"; do
+    git rev-parse --verify "$origin/$base" &> /dev/null || continue
+    commits_ahead=$(git log --oneline "$origin/$base..HEAD" | wc -l)
+    if [ "$min_diff" -eq -1 ] || [ "$min_diff" -gt "$commits_ahead" ]; then
+        TARGET="$base"
+        min_diff=$commits_ahead
+    fi
+done
+echo "$TARGET"
+exit 0