DAOS-14514 pmdk: PMDK release 2.1.0 (w/o NDCTL enabled) (#37)

Upgrade to version 2.0.1.
This version reduces libpmemobj's stack usage below the 11kB threshold.

This is the version that allows enabling NDCTL without
risk of stack over-usage in argobots ULT.

Extend symbols with new API calls.

Remove libpmem2 library from the installation.

Skip-list-master: test_ior_intercept_libioil:DAOS-16260 test_osa_offline_reintegration_without_checksum:DAOS-15608 test_daos_drain_simple:DAOS-15271 test_daos_rebuild_ec:DAOS-14982 test_osa_offline_reintegration_without_checksum:DAOS-14570
Skip-list-release/2.6: test_cart_rpc:DAOS-15989 test_daos_rebuild_ec:DAOS-14982 test_dfuse_daos_build_wt_pil4dfs:DAOS-16215 test_daos_rebuild_simple:DAOS-15290 test_dfuse_daos_build_wb:DAOS-16215 test_dfuse_daos_build_wt:DAOS-16215 test_dfuse_daos_build_metadata:DAOS-16215 test_dfuse_daos_build_data:DAOS-16215 test_dfuse_daos_build_nocache:DAOS-16215
Allow-unstable-test: true
Skip-func-hw-test-large: false

Signed-off-by: Tomasz Gromadzki <[email protected]>
Signed-off-by: Oksana Salyk <[email protected]>
Signed-off-by: Jan Michalski <[email protected]>
Co-authored-by: Oksana Sałyk <[email protected]>
Co-authored-by: Jan Michalski <[email protected]>

Jenkinsfile

@@ -39,4 +39,5 @@
 // To use a test branch (i.e. PR) until it lands to master
 // I.e. for testing library changes
 //@Library(value="pipeline-lib@your_branch") _
-packageBuildingPipelineDAOSTest(['distros': ['centos7', 'el8', 'el9', 'leap15', 'ubuntu20.04'], 'test-tag': 'daosio'])
+packageBuildingPipelineDAOSTest(['distros': ['el8', 'el9', 'leap15', 'ubuntu20.04'],
+				 'test-tag': 'pr daosio'])

Makefile

@@ -3,6 +3,4 @@ SRC_EXT       := gz
 TEST_PACKAGES := libpmem libpmem-devel libpmemobj libpmemobj-devel libpmempool \
 		 libpmempool-devel pmempool pmreorder
 
-EXTERNAL_RPM_BUILD_OPTIONS := --without=ndctl
-
 include packaging/Makefile_packaging.mk

debian/README.Debian

@@ -17,6 +17,6 @@ pmempool is a management tool for persistent memory pool files created by the
 NVM libraries. It may be useful for system administrators as well as for
 software developers for troubleshooting and debugging.
 
-Further details are at http://pmem.io/.
+Further details are at https://github.com/pmem/pmdk/.
 
  -- Nishanth Aravamudan <[email protected]>  Thu, 08 Sep 2016 15:19:46 -0700

debian/changelog

@@ -1,3 +1,25 @@
+pmdk (2.1.0-1) stable; urgency=medium
+
+  * Update to release 2.1.0 w/o NDCTL support which:
+    * Introduces the new logging subsystem in the release build for all libraries.
+    * Messages by default are printed to syslog and stderr but might be redirected to a user-defined function, see pmem(obj)_log_set_function() for details.
+    * Log level thresholds are controlled via new API, see pmem(obj)_log_set_treshold() for details.
+    * These new APIs are not available for LIBPMEMPOOL at the moment.
+    * The new logging subsystem is suppressed in the debug build when any of the legacy debug logging environment variables is set:
+      * PMEM_LOG_LEVEL/_FILE
+      * PMEMOBJ_LOG_LEVEL/_FILE
+      * PMEMPOOL_LOG_LEVEL/_FILE
+      * The debug logging subsystem becomes DEPRECATED.
+    * Drops support for building without libpthread (NO_LIBPTHREAD build-time define).
+    * Introduces fuses against ill-considered use of NDCTL_ENABLE=n.
+      * PMEMOBJ_IGNORE_DIRTY_SHUTDOWN and PMEMOBJ_IGNORE_BAD_BLOCKS are required to acknowledge the understanding of what production-critical functions are missing for the build without NDCTL.
+    * Does not allow create PMEMOBJ pool without unsafe shutdown counter (USC) if not explicitly disabled.
+      * use PMEMOBJ_CONF="sds.at_create=0" to disable USC when working without PMem (emulated PMem, Docker, etc.).
+  * Includes also release 2.0.1 which
+    * Reduces libpmemobj's stack usage below the 11kB threshold.
+
+ -- Tomasz Gromadzki <[email protected]>  Tue, 06 Aug 2024 10:00:00 +0000
+
 pmdk (2.0.0-1) stable; urgency=medium
 
   * Upgrade to 2.0.0 which:

debian/control

@@ -4,11 +4,10 @@ Priority: optional
 Maintainer: Adam Borowski <[email protected]>
 Build-Depends: bash-completion,
                bc,
-               cmake,
                debhelper-compat (= 12),
                dh-exec,
-               libdaxctl-dev (>= 64),
-               libndctl-dev (>= 64),
+               libdaxctl-dev (>= 63),
+               libndctl-dev (>= 63),
                pandoc,
                pkg-config,
                python3,
@@ -18,7 +17,7 @@ Build-Depends: bash-completion,
                ed
 Standards-Version: 4.5.0
 Rules-Requires-Root: no
-Homepage: https://pmem.io/pmdk/
+Homepage: https://github.com/pmem/pmdk/
 Vcs-Git: https://github.com/kilobyte/pmdk -b debian
 Vcs-Browser: https://github.com/kilobyte/pmdk/tree/debian
 

debian/libpmem1.symbols

@@ -11,6 +11,9 @@ libpmem.so.1 libpmem1 #MINVER#
  pmem_has_auto_flush@LIBPMEM_1.0 1.4
  pmem_has_hw_drain@LIBPMEM_1.0 1.4
  pmem_is_pmem@LIBPMEM_1.0 1.4
+ pmem_log_get_threshold@LIBPMEM_1.0 1.4
+ pmem_log_set_function@LIBPMEM_1.0 1.4
+ pmem_log_set_threshold@LIBPMEM_1.0 1.4
  pmem_map_file@LIBPMEM_1.0 1.4
  pmem_memcpy@LIBPMEM_1.0 1.5
  pmem_memcpy_nodrain@LIBPMEM_1.0 1.4
@@ -24,3 +27,4 @@ libpmem.so.1 libpmem1 #MINVER#
  pmem_msync@LIBPMEM_1.0 1.4
  pmem_persist@LIBPMEM_1.0 1.4
  pmem_unmap@LIBPMEM_1.0 1.4
+

debian/libpmemobj1.symbols

@@ -32,6 +32,9 @@ libpmemobj.so.1 libpmemobj1 #MINVER#
  pmemobj_list_insert_new@LIBPMEMOBJ_1.0 1.4
  pmemobj_list_move@LIBPMEMOBJ_1.0 1.4
  pmemobj_list_remove@LIBPMEMOBJ_1.0 1.4
+ pmemobj_log_get_threshold@LIBPMEMOBJ_1.0 1.4
+ pmemobj_log_set_function@LIBPMEMOBJ_1.0 1.4
+ pmemobj_log_set_threshold@LIBPMEMOBJ_1.0 1.4
  pmemobj_memcpy@LIBPMEMOBJ_1.0 1.5
  pmemobj_memcpy_persist@LIBPMEMOBJ_1.0 1.4
  pmemobj_memset_persist@LIBPMEMOBJ_1.0 1.4

debian/rules

@@ -5,16 +5,24 @@ export DEB_BUILD_MAINT_OPTIONS = hardening=+all
 DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH)
 DEB_UPSTREAM_VERSION = $(shell dpkg-parsechangelog -S version | cut -d - -f 1 | cut -d : -f 2)
 
-BUILD_ARGS = prefix=/usr libdir=/usr/lib/$(DEB_HOST_MULTIARCH) sysconfdir=/etc bashcompdir=/usr/share/bash-completion/completions NORPATH=1 NDCTL_ENABLE=n BUILD_EXAMPLES=n BUILD_BENCHMARKS=n
+BUILD_ARGS = prefix=/usr libdir=/usr/lib/$(DEB_HOST_MULTIARCH) sysconfdir=/etc bashcompdir=/usr/share/bash-completion/completions NORPATH=1 BUILD_EXAMPLES=n BUILD_BENCHMARKS=n NDCTL_ENABLE=n PMEMOBJ_IGNORE_DIRTY_SHUTDOWN=y PMEMOBJ_IGNORE_BAD_BLOCKS=y
 
 %:
 	#dh $@ --with bash-completion
 	dh $@
-
+
+# !!!!! NOTICE !!!!!
+#
+# `dh_auto_clean` MUST BE CALLED WITH `NDCTL_ENABLE=n`
+#
+# It is crucial for build process to run `dh_auto_clean` with NDCTL disabled (NDCTL_ENABLE=n).
+#
+# `db_auto_clean` calls `make -j72 distclean` which requires NDCTL when NDCTL_ENABLE<>n.
+# This dependency (libndctl, libdaxctl) is resolved after `db_auto_clean` step.
+#
 override_dh_auto_clean:
-	dh_auto_clean -- $(BUILD_ARGS)
+	dh_auto_clean -- $(BUILD_ARGS) NDCTL_ENABLE=n PMEMOBJ_IGNORE_DIRTY_SHUTDOWN=y PMEMOBJ_IGNORE_BAD_BLOCKS=y
 	rm -rf src/test/unittest/__pycache__
-	rm -rf src/deps/miniasync/build
 
 override_dh_auto_build:
 	+$(MAKE) doc
@@ -29,7 +37,7 @@ override_dh_install:
 	dh_install
 
 override_dh_installexamples:
-	dh_installexamples --exclude=.gitignore --exclude=.vcxproj
+	dh_installexamples --exclude=.gitignore
 
 override_dh_installman:
 	dh_installman

packaging/Dockerfile.centos.7

@@ -5,19 +5,41 @@
 #
 
 # Pull base image
-FROM centos:7
+FROM centos:centos7
 LABEL maintainer="[email protected]"
 
+# Use local repo server if present
+ARG REPO_FILE_URL
+RUN set -e;                                                      \
+    if [ -n "$REPO_FILE_URL" ]; then                             \
+        cd /etc/yum.repos.d/ &&                                  \
+        curl -k -f -o daos_ci-centos7-artifactory.repo.tmp       \
+             "$REPO_FILE_URL"daos_ci-centos7-artifactory.repo && \
+        for file in *.repo; do                                   \
+            true > $file;                                        \
+        done;                                                    \
+        mv daos_ci-centos7-artifactory.repo{.tmp,};              \
+    fi;                                                          \
+    yum -y install dnf;                                          \
+    yum clean all;                                               \
+    dnf --disablerepo \*epel\* -y install epel-release           \
+                                          dnf-plugins-core;      \
+    if [ -n "$REPO_FILE_URL" ]; then                             \
+        dnf -y --quiet config-manager --disable epel;            \
+    fi;                                                          \
+    dnf -y update epel-release;                                  \
+    dnf -y clean all
+
 # use same UID as host and default value of 1000 if not specified
 ARG UID=1000
 
 # Update distribution
 #Nothing to do for CentOS
 
 # Install basic tools
-RUN yum install -y epel-release
-RUN yum install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \
-                   git python-srpm-macros dnf
+RUN dnf install -y epel-release
+RUN dnf install -y mock make rpm-build curl createrepo rpmlint redhat-lsb-core \
+                   git python-srpm-macros dnf && dnf -y clean all
 
 # Add build user (to keep rpmbuild happy)
 ENV USER build

packaging/Dockerfile.mockbuild

@@ -1,11 +1,11 @@
 #
-# Copyright 2018-2023 Intel Corporation
+# Copyright 2018-2024 Intel Corporation
 #
 # 'recipe' for Docker to build an RPM
 #
 
 # Pull base image
-ARG FVERSION=38
+ARG FVERSION=latest
 FROM fedora:$FVERSION
 # Needed for later use of FVERSION
 ARG FVERSION
@@ -15,27 +15,32 @@ LABEL maintainer="[email protected]"
 ARG REPO_FILE_URL
 RUN if [ -n "$REPO_FILE_URL" ]; then                            \
         cd /etc/yum.repos.d/ &&                                 \
-        curl -f -o daos_ci-fedora-artifactory.repo.tmp          \
+        curl -k -f -o daos_ci-fedora-artifactory.repo.tmp       \
              "$REPO_FILE_URL"daos_ci-fedora-artifactory.repo && \
-        rm -f *.repo &&                                         \
+        for file in *.repo; do                                  \
+            true > $file;                                       \
+        done;                                                   \
         mv daos_ci-fedora-artifactory.repo{.tmp,};              \
     fi
 
 # Install basic tools
 RUN dnf -y install mock make                                        \
                    rpm-build createrepo rpmlint redhat-lsb-core git \
-                   python-srpm-macros rpmdevtools
+                   python-srpm-macros rpmdevtools &&                \
+    dnf -y clean all
 
 # use same UID as host and default value of 1000 if not specified
 ARG UID=1000
 
 # Add build user (to keep rpmbuild happy)
 ENV USER build
 ENV PASSWD build
-RUN useradd -u $UID -ms /bin/bash $USER
-RUN echo "$USER:$PASSWD" | chpasswd
 # add the user to the mock group so it can run mock
-RUN usermod -a -G mock $USER
+RUN if [ $UID != 0 ]; then                   \
+        useradd -u $UID -ms /bin/bash $USER; \
+        echo "$USER:$PASSWD" | chpasswd;     \
+        usermod -a -G mock $USER;            \
+    fi
 
 ARG CB0
 RUN dnf -y upgrade && \
@@ -45,15 +50,24 @@ RUN dnf -y upgrade && \
 # https://github.com/rpm-software-management/rpmlint/pull/795 in it
 # But make sure to patch after dnf upgrade so that an upgraded rpmlint
 # RPM doesn't wipe out our patch
-COPY packaging/rpmlint--ignore-unused-rpmlintrc.patch .
+# Ditto for the patch to zero and display ccache stats
+# https://github.com/rpm-software-management/mock/pull/1299
+ARG PACKAGINGDIR=packaging
+COPY ${PACKAGINGDIR}/*.patch ./
 RUN (cd $(python3 -c 'import site; print(site.getsitepackages()[-1])') &&                      \
      if ! grep -e --ignore-unused-rpmlintrc rpmlint/cli.py; then                               \
-         if ! patch -p1; then                                                                  \
+         if ! patch -p1 < $OLDPWD/rpmlint--ignore-unused-rpmlintrc.patch; then                 \
              exit 1;                                                                           \
          fi;                                                                                   \
          rm -f rpmlint/__pycache__/{cli,lint}.*.pyc;                                           \
-     fi) < rpmlint--ignore-unused-rpmlintrc.patch;                                             \
-    rm -f  rpmlint--ignore-unused-rpmlintrc.patch
+     fi;                                                                                       \
+     if ! grep _ccachePostBuildHook mockbuild/plugins/ccache.py; then                          \
+         if ! patch -p3 < $OLDPWD/ccache-stats.patch; then                                     \
+             exit 1;                                                                           \
+         fi;                                                                                   \
+         rm -f mockbuild/plugins/__pycache__/ccache.*.pyc;                                     \
+     fi);                                                                                      \
+     rm -f rpmlint--ignore-unused-rpmlintrc.patch ccache-stats.patch
 
 # show the release that was built
 ARG CACHEBUST

packaging/Dockerfile.ubuntu.20.04

@@ -18,26 +18,23 @@ RUN if [ -n "$REPO_FILE_URL" ]; then                                 \
         true > ../sources.list &&                                    \
         mv daos_ci-ubuntu20.04-artifactory.list.tmp                  \
            daos_ci-ubuntu20.04-artifactory.list;                     \
+        url="${REPO_FILE_URL%/*/}/hpe-ilorest-ubuntu-bionic-proxy/"; \
+    else                                                             \
+        url="https://downloads.linux.hpe.com/SDR/repo/ilorest/";     \
     fi;                                                              \
     cd -;                                                            \
-    curl -f -O "$REPO_FILE_URL"esad_repo.key;                        \
-    gpg --no-default-keyring --keyring ./temp-keyring.gpg            \
-        --import esad_repo.key;                                      \
     mkdir -p /usr/local/share/keyrings/;                             \
+    curl -f -O "$url"GPG-KEY-hprest;                                 \
+    gpg --no-default-keyring --keyring ./temp-keyring.gpg            \
+        --import GPG-KEY-hprest;                                     \
     gpg --no-default-keyring --keyring ./temp-keyring.gpg --export   \
-        --output /usr/local/share/keyrings/daos-stack-public.gpg;    \
+        --output /usr/local/share/keyrings/hpe-sdr-public.gpg;       \
     rm ./temp-keyring.gpg;                                           \
-    url_prefix=https://downloads.linux.hpe.com/SDR/;                 \
-    for url in hpPublicKey2048.pub                                   \
-               hpPublicKey2048_key1.pub                              \
-               hpePublicKey2048_key1.pub; do                         \
-        curl -f -O "$url_prefix$url";                                \
-        gpg --no-default-keyring --keyring ./temp-keyring.gpg        \
-            --import "$(basename $url)";                             \
-    done;                                                            \
+    curl -f -O "$REPO_FILE_URL"esad_repo.key;                        \
+    gpg --no-default-keyring --keyring ./temp-keyring.gpg            \
+        --import esad_repo.key;                                      \
     gpg --no-default-keyring --keyring ./temp-keyring.gpg --export   \
-        --output /usr/local/share/keyrings/hpe-sdr-public.gpg;       \
-    rm ./temp-keyring.gpg
+        --output /usr/local/share/keyrings/daos-stack-public.gpg
 
 # Install basic tools
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \

packaging/Makefile_distro_vars.mk

@@ -83,6 +83,14 @@ DISTRO_VERSION  ?= $(VERSION_ID)
 ORIG_TARGET_VER := 15.4
 SED_EXPR        := 1p
 endif
+ifeq ($(CHROOT_NAME),opensuse-leap-15.5-x86_64)
+VERSION_ID      := 15.5
+DISTRO_ID       := sl15.5
+DISTRO_BASE     := LEAP_15
+DISTRO_VERSION  ?= $(VERSION_ID)
+ORIG_TARGET_VER := 15.5
+SED_EXPR        := 1p
+endif
 endif
 ifeq ($(ID),centos)
 ID = el