DAOS-15079 test: VMD Hot Plug - Support rpc.py in spdk-tools package

Jenkinsfile

@@ -41,5 +41,5 @@
 //@Library(value="pipeline-lib@your_branch") _
 
 /* groovylint-disable-next-line CompileStatic */
-packageBuildingPipelineDAOSTest(['distros': ['centos7', 'el8', 'el9', 'leap15', 'ubuntu20.04'],
+packageBuildingPipelineDAOSTest(['distros': ['el8', 'el9', 'leap15', 'ubuntu20.04'],
                                 'test-tag': 'pr'])

debian/changelog

@@ -1,3 +1,11 @@
+spdk (22.01.2-6) unstable; urgency=medium
+
+  [ Tomasz Gromadzki ]
+  * Add rpc.py to spdk-tools package.
+  * Synchronize Debian package with RPMs by adding examples to spdk-tools package.
+
+ --  Tomasz Gromadzki <[email protected]>  Tue, 12 Mar 2024 00:00:00 +0000
+
 spdk (22.01.2-3) unstable; urgency=medium
 
   [ Tom Nabarro ]

debian/control

@@ -3,7 +3,7 @@ Priority: optional
 Maintainer: daos-stack <[email protected]>
 Build-Depends:
  debhelper,
- python,
+ python3,
  libdpdk-dev (>= 21.11.2),
  librdmacm-dev,
  libibverbs-dev,

debian/libspdk-dev.install

@@ -1,4 +1,4 @@
 usr/include/*
 lib/x86_64-linux-gnu/lib*.so
 lib/x86_64-linux-gnu/lib*.a
-lib/x86_64-linux-gnu/pkgconfig/*
+lib/x86_64-linux-gnu/pkgconfig/*

debian/libspdk3.install

@@ -1 +1 @@
-lib/x86_64-linux-gnu/lib*.so.*
+lib/x86_64-linux-gnu/lib*.so.*

debian/rules

@@ -6,6 +6,13 @@ export DEB_BUILD_MAINT_OPTIONS = hardening=+all
 # https://github.com/spdk/spdk/issues/2011
 export DEB_CFLAGS_MAINT_APPEND = $(shell pkg-config --cflags libdpdk)
 
+prefix := /usr
+
+cflags := -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 \
+	-Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong \
+	-grecord-gcc-switches -m64 -mtune=generic -fasynchronous-unwind-tables \
+	-fstack-clash-protection -fcf-protection
+
 %:
 	dh $@
 
@@ -41,11 +48,17 @@ override_dh_auto_configure:
 	            --without-vtune           \
 	            --with-shared
 
-#override_dh_auto_build:
-#	make V=1 # -j 8
+override_dh_auto_build:
+	export CFLAGS='$(cflags)'
+	export CXXFLAGS='$(cflags)'
+	export FFLAGS='$(cflags)'
+	export LDFLAGS='-Wl,-z,relro -Wl,-z,now'
+	dh_auto_build -- CONFIG_ARCH=haswell V=1
 
 override_dh_auto_install:
-	dh_auto_install -- prefix=/usr libdir=/lib/x86_64-linux-gnu/ datadir=/usr/share
+	mv build/examples/lsvmd build/examples/spdk_lsvmd
+	mv build/examples/nvme_manage build/examples/spdk_nvme_manage
+	dh_auto_install -- prefix=$(prefix) libdir=/lib/x86_64-linux-gnu/ datadir=$(prefix)/share
 
 override_dh_missing:
 	dh_missing --fail-missing

debian/spdk-tools.dirs

@@ -0,0 +1,4 @@
+usr/share/spdk-tools/scripts
+usr/include/spdk
+usr/share/spdk-tools/python
+usr/share/spdk-tools/python/rpc

debian/spdk-tools.install

@@ -1 +1,7 @@
-usr/bin/*
+usr/bin/*
+scripts/{setup,common}.sh usr/share/spdk-tools/scripts
+include/spdk/pci_ids.h usr/include/spdk
+scripts/rpc.py usr/share/spdk-tools/python
+scripts/rpc/*.py usr/share/spdk-tools/python/rpc
+build/examples/spdk_lsvmd usr/bin/
+build/examples/spdk_nvme_manage usr/bin/

packaging/Dockerfile.mockbuild

@@ -1,11 +1,11 @@
 #
-# Copyright 2018-2023 Intel Corporation
+# Copyright 2018-2024 Intel Corporation
 #
 # 'recipe' for Docker to build an RPM
 #
 
 # Pull base image
-ARG FVERSION=38
+ARG FVERSION=latest
 FROM fedora:$FVERSION
 # Needed for later use of FVERSION
 ARG FVERSION
@@ -48,16 +48,24 @@ RUN dnf -y upgrade && \
 # https://github.com/rpm-software-management/rpmlint/pull/795 in it
 # But make sure to patch after dnf upgrade so that an upgraded rpmlint
 # RPM doesn't wipe out our patch
+# Ditto for the patch to zero and display ccache stats
+# https://github.com/rpm-software-management/mock/pull/1299
 ARG PACKAGINGDIR=packaging
-COPY ${PACKAGINGDIR}/rpmlint--ignore-unused-rpmlintrc.patch .
+COPY ${PACKAGINGDIR}/*.patch ./
 RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') &&                      \
      if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then                               \
-         if ! patch -p1; then                                                                  \
+         if ! patch -p1 < $OLDPWD/rpmlint--ignore-unused-rpmlintrc.patch; then                 \
              exit 1;                                                                           \
          fi;                                                                                   \
          rm -f rpmlint/__pycache__/{cli,lint}.*.pyc;                                           \
-     fi) < rpmlint--ignore-unused-rpmlintrc.patch;                                             \
-    rm -f  rpmlint--ignore-unused-rpmlintrc.patch
+     fi;                                                                                       \
+     if ! grep _ccachePostBuildHook mockbuild/plugins/ccache.py; then                          \
+         if ! patch -p3 < $OLDPWD/ccache-stats.patch; then                                     \
+             exit 1;                                                                           \
+         fi;                                                                                   \
+         rm -f mockbuild/plugins/__pycache__/ccache.*.pyc;                                     \
+     fi);                                                                                      \
+     rm -f rpmlint--ignore-unused-rpmlintrc.patch ccache-stats.patch
 
 # show the release that was built
 ARG CACHEBUST

packaging/Makefile_distro_vars.mk

@@ -83,6 +83,14 @@ DISTRO_VERSION  ?= $(VERSION_ID)
 ORIG_TARGET_VER := 15.4
 SED_EXPR        := 1p
 endif
+ifeq ($(CHROOT_NAME),opensuse-leap-15.5-x86_64)
+VERSION_ID      := 15.5
+DISTRO_ID       := sl15.5
+DISTRO_BASE     := LEAP_15
+DISTRO_VERSION  ?= $(VERSION_ID)
+ORIG_TARGET_VER := 15.5
+SED_EXPR        := 1p
+endif
 endif
 ifeq ($(ID),centos)
 ID = el

packaging/Makefile_packaging.mk

@@ -54,7 +54,8 @@ RPM_BUILD_OPTIONS := $(BUILD_DEFINES)
 GIT_DIFF_EXCLUDES := $(PATCH_EXCLUDE_FILES:%=':!%')
 endif
 
-COMMON_RPM_ARGS   = --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES)
+FVERSION         ?= latest
+COMMON_RPM_ARGS  := --define "_topdir $$PWD/_topdir" $(BUILD_DEFINES)
 SPEC             := $(shell if [ -f $(NAME)-$(DISTRO_BASE).spec ]; then echo $(NAME)-$(DISTRO_BASE).spec; else echo $(NAME).spec; fi)
 VERSION           = $(eval VERSION := $(shell rpm $(COMMON_RPM_ARGS) --specfile --qf '%{version}\n' $(SPEC) | sed -n '1p'))$(VERSION)
 DEB_RVERS        := $(subst $(DOT),\$(DOT),$(VERSION))
@@ -363,12 +364,14 @@ chrootbuild: $(SRPM) $(CALLING_MAKEFILE)
 	LOCAL_REPOS='$(LOCAL_REPOS)'                            \
 	ARTIFACTORY_URL="$(ARTIFACTORY_URL)"                    \
 	DISTRO_VERSION="$(DISTRO_VERSION)"                      \
+	PACKAGE="$(NAME)"                                       \
 	TARGET="$<"                                             \
 	packaging/rpm_chrootbuild
 endif
 
 podman_chrootbuild:
 	if ! podman build --build-arg REPO_FILE_URL=$(REPO_FILE_URL) \
+	                  --build-arg FVERSION=$(FVERSION)           \
 	                  -t $(subst +,-,$(CHROOT_NAME))-chrootbuild \
 	                  -f packaging/Dockerfile.mockbuild .; then  \
 		echo "Container build failed";                           \
@@ -386,7 +389,9 @@ podman_chrootbuild:
 	                                 exit 1;                                                                                        \
 	                             fi;                                                                                                \
 	                             rpmlint $$(ls /var/lib/mock/$(CHROOT_NAME)/result/*.rpm |                                          \
-	                                 grep -v -e debuginfo -e debugsource -e src.rpm)'
+	                                 grep -v -e debuginfo -e debugsource -e src.rpm)'; then                                         \
+		exit 1;                                                                                                                 \
+	fi
 
 docker_chrootbuild:
 	if ! $(DOCKER) build --build-arg UID=$$(id -u) -t chrootbuild   \

packaging/ccache-stats.patch

@@ -0,0 +1,66 @@
+From e87d916d7f49ea4949973adf0f09e9e5bf891e03 Mon Sep 17 00:00:00 2001
+From: "Brian J. Murrell" <[email protected]>
+Date: Tue, 30 Jan 2024 11:03:12 -0500
+Subject: [PATCH 1/2] Show ccache stats at the end of the build
+
+Zero the ccache stats at the beginning of the build and then display the
+ccache stats at the end of the build to see how effective ccache was.
+
+Signed-off-by: Brian J. Murrell <[email protected]>
+---
+ mock/py/mockbuild/plugins/ccache.py | 11 +++++++++++
+ 1 file changed, 11 insertions(+)
+
+diff --git a/mock/py/mockbuild/plugins/ccache.py b/mock/py/mockbuild/plugins/ccache.py
+index 2666ad9fc..1080ffe68 100644
+--- a/mock/py/mockbuild/plugins/ccache.py
++++ b/mock/py/mockbuild/plugins/ccache.py
+@@ -35,6 +35,7 @@ def __init__(self, plugins, conf, buildroot):
+         buildroot.preexisting_deps.append("ccache")
+         plugins.add_hook("prebuild", self._ccacheBuildHook)
+         plugins.add_hook("preinit", self._ccachePreInitHook)
++        plugins.add_hook("postbuild", self._ccachePostBuildHook)
+         buildroot.mounts.add(
+             BindMountPoint(srcpath=self.ccachePath, bindpath=buildroot.make_chroot_path("/var/tmp/ccache")))
+
+@@ -47,6 +48,9 @@ def __init__(self, plugins, conf, buildroot):
+     @traceLog()
+     def _ccacheBuildHook(self):
+         self.buildroot.doChroot(["ccache", "-M", str(self.ccache_opts['max_cache_size'])], shell=False)
++        # zero ccache stats
++        getLog().info("Zero ccache stats:")
++        self.buildroot.doChroot(["ccache", "--zero-stats"], printOutput=True, shell=False)
+
+     # set up the ccache dir.
+     # we also set a few variables used by ccache to find the shared cache.
+@@ -61,3 +65,10 @@ def _ccachePreInitHook(self):
+         file_util.mkdirIfAbsent(self.buildroot.make_chroot_path('/var/tmp/ccache'))
+         file_util.mkdirIfAbsent(self.ccachePath)
+         self.buildroot.uid_manager.changeOwner(self.ccachePath, recursive=True)
++
++    # get some cache stats
++    def _ccachePostBuildHook(self):
++        # show the cache hit stats
++        getLog().info("ccache stats:")
++        self.buildroot.doChroot(["ccache", "--show-stats"], printOutput=True, shell=False)
+++
+
+From bfd3a7e1bb47d28ee60a94cb5985c1f66476475f Mon Sep 17 00:00:00 2001
+From: "Brian J. Murrell" <[email protected]>
+Date: Tue, 30 Jan 2024 11:17:48 -0500
+Subject: [PATCH 2/2] Remove extraneous line
+
+Signed-off-by: Brian J. Murrell <[email protected]>
+---
+ mock/py/mockbuild/plugins/ccache.py | 1 -
+ 1 file changed, 1 deletion(-)
+
+diff --git a/mock/py/mockbuild/plugins/ccache.py b/mock/py/mockbuild/plugins/ccache.py
+index 1080ffe68..1a20846d3 100644
+--- a/mock/py/mockbuild/plugins/ccache.py
++++ b/mock/py/mockbuild/plugins/ccache.py
+@@ -71,4 +71,3 @@ def _ccachePostBuildHook(self):
+         # show the cache hit stats
+         getLog().info("ccache stats:")
+         self.buildroot.doChroot(["ccache", "--show-stats"], printOutput=True, shell=False)
+-+

packaging/get_base_branch

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# find the base branch of the current branch
+
+set -eux -o pipefail
+IFS=' ' read -r -a add_bases <<< "${1:-}"
+origin=origin
+mapfile -t all_bases < <(echo "master"
+                         git branch -r | sed -ne "/^  $origin\\/release\\/[0-9]/s/^  $origin\\///p")
+all_bases+=("${add_bases[@]}")
+TARGET="master"
+min_diff=-1
+for base in "${all_bases[@]}"; do
+    git rev-parse --verify "$origin/$base" &> /dev/null || continue
+    commits_ahead=$(git log --oneline "$origin/$base..HEAD" | wc -l)
+    if [ "$min_diff" -eq -1 ] || [ "$min_diff" -gt "$commits_ahead" ]; then
+        TARGET="$base"
+        min_diff=$commits_ahead
+    fi
+done
+echo "$TARGET"
+exit 0

packaging/rpm_chrootbuild

@@ -2,10 +2,14 @@
 
 set -uex
 
-original_cfg_file="/etc/mock/$CHROOT_NAME.cfg"
-cfg_file=mock.cfg
+cp /etc/mock/"$CHROOT_NAME".cfg mock.cfg
+
+# Enable mock ccache plugin
+cat <<EOF >> mock.cfg
+config_opts['plugin_conf']['ccache_enable'] = True
+config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/"
+EOF
 
-cp "$original_cfg_file" "$cfg_file"
 
 if [[ $CHROOT_NAME == *epel-8-x86_64 ]]; then
     cat <<EOF >> mock.cfg
@@ -22,7 +26,7 @@ if [[ $CHROOT_NAME == *epel-7-x86_64 ]]; then
 fi
 
 # Allow BR: foo-devel < 1.2 to work when foo-devel-1.3 is actually available
-cat <<EOF >> "$cfg_file"
+cat <<EOF >> mock.cfg
 config_opts['dnf.conf'] += """
 [main]
 best=0
@@ -33,7 +37,7 @@ EOF
 repo_adds=()
 repo_dels=()
 
-echo -e "config_opts['yum.conf'] += \"\"\"\n" >> "$cfg_file"
+echo -e "config_opts['yum.conf'] += \"\"\"\n" >> mock.cfg
 
 if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then
     repo_dels+=("--disablerepo=\*")
@@ -56,7 +60,7 @@ if [ -n "${ARTIFACTORY_URL:-}" ] && "$LOCAL_REPOS"; then
                 REPO_FILE_URL="file://$(readlink -e "$REPO_FILES_PR")/"
             fi
         fi
-        curl -sSf "${REPO_FILE_URL}daos_ci-$DISTRO"-mock-artifactory.repo >> "$cfg_file"
+        curl -sSf "$REPO_FILE_URL"daos_ci-"${CHROOT_NAME%-*}".repo >> mock.cfg
         repo_adds+=("--enablerepo *-artifactory")
     fi
 fi
@@ -81,9 +85,9 @@ for repo in $DISTRO_BASE_PR_REPOS $PR_REPOS; do
     repo_adds+=("--enablerepo $repo:${branch//[@\/]/_}:$build_number")
     echo -e "[$repo:${branch//[@\/]/_}:$build_number]\n\
 name=$repo:${branch//[@\/]/_}:$build_number\n\
-baseurl=${JENKINS_URL:-https://build.hpdd.intel.com/}job/daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\
+baseurl=${ARTIFACTS_URL:-${JENKINS_URL:-https://build.hpdd.intel.com/}job/}daos-stack/job/$repo/job/${branch//\//%2F}/$build_number/artifact/artifacts/$DISTRO/\n\
 enabled=1\n\
-gpgcheck=False\n" >> "$cfg_file"
+gpgcheck=False\n" >> mock.cfg
 done
 for repo in $JOB_REPOS; do
     repo_name=${repo##*://}
@@ -97,29 +101,40 @@ for repo in $JOB_REPOS; do
     echo -e "[${repo_name//[@\/]/_}]\n\
 name=${repo_name}\n\
 baseurl=${repo//\//%2F}\n\
-enabled=1\n" >> "$cfg_file"
+enabled=1\n" >> mock.cfg
 done
-echo "\"\"\"" >> "$cfg_file"
+echo "\"\"\"" >> mock.cfg
 
 if [ -n "$DISTRO_VERSION" ]; then
     releasever_opt=("--config-opts=releasever=$DISTRO_VERSION")
 fi
 
 bs_dir=/scratch/mock/cache/"${CHROOT_NAME}"-bootstrap
-if ls -l /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz; then
-    mkdir -p "/var/cache/mock/${CHROOT_NAME}-bootstrap"
+if ls -l "$bs_dir"/root_cache/cache.tar.gz; then
+    mkdir -p "/var/cache/mock/${CHROOT_NAME}-bootstrap/"
     flock "$bs_dir" -c "cp -a $bs_dir/root_cache /var/cache/mock/${CHROOT_NAME}-bootstrap"
 fi
+if ls -l "$bs_dir/ccache-$CHROOT_NAME-$PACKAGE".tar.gz; then
+    flock "$bs_dir" -c "tar -C / -xzf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz"
+fi
 
-# shellcheck disable=SC2086
-eval mock -r "$cfg_file" ${repo_dels[*]} ${repo_adds[*]} --disablerepo=\*-debug* \
-     "${releasever_opt[@]}" $MOCK_OPTIONS $RPM_BUILD_OPTIONS "$TARGET"
+rc=0
+# shellcheck disable=SC2086,SC2048
+if ! eval time mock -r mock.cfg ${repo_dels[*]} ${repo_adds[*]} --no-clean \
+                    --disablerepo=\*-debug* ${releasever_opt[*]} $MOCK_OPTIONS  \
+                    $RPM_BUILD_OPTIONS "$TARGET"; then
+    rc=${PIPESTATUS[0]}
+fi
 
-date
-if ls -l /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz &&
-   [ -d /scratch/ ]; then
-    mkdir -p /scratch/mock/cache/"${CHROOT_NAME}"-bootstrap/
-    if ! cmp /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz "$bs_dir"/root_cache/cache.tar.gz; then
-        flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/"
+# Save the ccache
+if [ -d /scratch/ ]; then
+    mkdir -p "$bs_dir"/
+    flock "$bs_dir" -c "tar -czf $bs_dir/ccache-$CHROOT_NAME-$PACKAGE.tar.gz /var/cache/mock/${CHROOT_NAME}/ccache"
+    if ls -l /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz; then
+        if ! cmp /var/cache/mock/"${CHROOT_NAME}"-bootstrap/root_cache/cache.tar.gz "$bs_dir"/root_cache/cache.tar.gz; then
+            flock "$bs_dir" -c "cp -a /var/cache/mock/${CHROOT_NAME}-bootstrap/root_cache $bs_dir/"
+        fi
     fi
 fi
+
+exit "$rc"