From c27c3d8153d4b14fdaffac24b2cdbf463361e8d8 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Wed, 29 Nov 2023 23:30:37 +0100 Subject: [PATCH] DAOS-14410 common: update PMDK to version 2.0.0 (#13217) Update DAOS to use the latest stable version of PMDK. Remove irrelevant libpmem.h include. Signed-off-by: Tomasz Gromadzki --- .../post_provision_config_common_functions.sh | 4 ++-- .../post_provision_config_nodes_UBUNTU_20_04.sh | 2 +- debian/changelog | 13 ++++++++++++- debian/control | 2 +- docs/admin/hardware.md | 2 +- docs/admin/troubleshooting.md | 2 +- docs/overview/terminology.md | 2 +- site_scons/components/__init__.py | 4 ++-- src/vos/README.md | 4 ++-- src/vos/vos_dtx.c | 1 - utils/build.config | 2 +- utils/rpms/daos.spec | 17 +++++++++++++---- 12 files changed, 37 insertions(+), 18 deletions(-) diff --git a/ci/provisioning/post_provision_config_common_functions.sh b/ci/provisioning/post_provision_config_common_functions.sh index 2f34349d84a..7fd2b4e3d25 100755 --- a/ci/provisioning/post_provision_config_common_functions.sh +++ b/ci/provisioning/post_provision_config_common_functions.sh @@ -266,11 +266,11 @@ post_provision_config_nodes() { if $CONFIG_POWER_ONLY; then rm -f "$REPOS_DIR"/*.hpdd.intel.com_job_daos-stack_job_*_job_*.repo - time dnf -y erase fio fuse ior-hpc mpich-autoload \ + time dnf -y erase fio fuse ior-hpc mpich-autoload \ ompi argobots cart daos daos-client dpdk \ fuse-libs libisa-l libpmemobj mercury mpich \ pmix protobuf-c spdk libfabric libpmem \ - libpmemblk munge-libs munge slurm \ + munge-libs munge slurm \ slurm-example-configs slurmctld slurm-slurmmd fi diff --git a/ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh b/ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh index dd3d49d015f..5547756b074 100755 --- a/ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh +++ b/ci/provisioning/post_provision_config_nodes_UBUNTU_20_04.sh @@ -8,7 +8,7 @@ post_provision_config_nodes() { # ompi argobots cart daos daos-client dpdk \ # fuse-libs libisa-l libpmemobj mercury mpich \ # pmix protobuf-c spdk libfabric libpmem \ - # libpmemblk munge-libs munge slurm \ + # munge-libs munge slurm \ # slurm-example-configs slurmctld slurm-slurmmd #fi codename=$(lsb_release -s -c) diff --git a/debian/changelog b/debian/changelog index 62ba07cf606..25a56320ef7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,15 @@ -daos (2.5.100-11) unstable; urgency=medium +daos (2.5.100-12) unstable; urgency=medium + [ Tomasz Gromadzki ] + * Update PMDK to 2.0.0 + * Remove libpmemblk from dependencies. + * Start using BUILD_EXAMPLES=n and BUILD_BENCHMARKS=n instead of patches. + * Stop using BUILD_RPMEM=n (removed) and NDCTL_DISABLE=y (invalid). + * Point https://github.com/pmem/pmdk as the main PMDK reference source. + NOTE: PMDK upgrade to 2.0.0 does not affect any API call used by DAOS. + libpmemobj (and libpmem) API stays unchanged. + + -- Tomasz Gromadzki Fri, 17 Nov 2023 12:52:00 -0400 + [ Jerome Soumagne ] * Bump mercury min version to 2.3.1 diff --git a/debian/control b/debian/control index 33e0e6b664c..16e4380ae99 100644 --- a/debian/control +++ b/debian/control @@ -18,7 +18,7 @@ Build-Depends: debhelper (>= 10), python3-distro, libabt-dev, libucx-dev, - libpmemobj-dev (>= 1.12.1~rc1), + libpmemobj-dev (>= 2.0.0), libfuse3-dev, libprotobuf-c-dev, libjson-c-dev, diff --git a/docs/admin/hardware.md b/docs/admin/hardware.md index 423acea6b81..3f4658cf40b 100644 --- a/docs/admin/hardware.md +++ b/docs/admin/hardware.md @@ -21,7 +21,7 @@ servers. DAOS requires a 64-bit processor architecture and is primarily developed on Intel x86\_64 architecture. The DAOS software and the libraries it depends on (e.g., [ISA-L](https://github.com/intel/isa-l), -[SPDK](https://pmem.io/pmdk/), [PMDK](https://spdk.io/), and +[SPDK](https://spdk.io/), [PMDK](https://github.com/pmem/pmdk/), and [DPDK](https://www.dpdk.org/) can take advantage of Intel Streaming SIMD (SSE) and Intel Advanced Vector Extensions (AVX). diff --git a/docs/admin/troubleshooting.md b/docs/admin/troubleshooting.md index df5532baa14..2116f5957e5 100644 --- a/docs/admin/troubleshooting.md +++ b/docs/admin/troubleshooting.md @@ -907,7 +907,7 @@ Please refer the [ndctl list](https://docs.pmem.io/ndctl-user-guide/ndctl-man-pa The pmempool is a management tool for Persistent Memory pool files created by PMDK libraries. DAOS uses the PMDK library to manage persistence inside ext4 files. -[pmempool](https://pmem.io/pmdk/manpages/linux/v1.9/pmempool/pmempool-check.1/) can check consistency of a given pool file. +[pmempool](https://github.com/pmem/pmdk/blob/stable-2.0/doc/pmempool/pmempool-check.1.md) can check consistency of a given pool file. It can be run with -r (repair) option which can fix some of the issues with pool file. DAOS will have more number of such pool file (vos-*), based on number of targets mention per daos engine. User may need to check each vos pool file for corruption on faulty pool. diff --git a/docs/overview/terminology.md b/docs/overview/terminology.md index 47fb6b21a67..a9d52958961 100644 --- a/docs/overview/terminology.md +++ b/docs/overview/terminology.md @@ -32,7 +32,7 @@ |[OFI](https://ofiwg.github.io/libfabric/)|Open Fabrics Interfaces| |OS|Operating System| |PM|Persistent Memory| -|[PMDK](https://pmem.io/pmdk/)|Persistent Memory Devevelopment Kit| +|[PMDK](https://github.com/pmem/pmdk)|Persistent Memory Devevelopment Kit| |[RAFT](https://raft.github.io/)|Raft is a consensus algorithm used to distribute state transitions among DAOS server nodes.| |RAS|Reliability, Availability & Serviceability| |RDB|Replicated Database, containing pool metadata and maintained across DAOS servers using the Raft algorithm.| diff --git a/site_scons/components/__init__.py b/site_scons/components/__init__.py index ede55290b73..a338cd3e6fe 100644 --- a/site_scons/components/__init__.py +++ b/site_scons/components/__init__.py @@ -266,9 +266,9 @@ def define_components(reqs): retriever=GitRepoRetriever('https://github.com/pmem/pmdk.git'), commands=[['make', 'all', - 'BUILD_RPMEM=n', 'NDCTL_ENABLE=n', - 'NDCTL_DISABLE=y', + 'BUILD_EXAMPLES=n', + 'BUILD_BENCHMARKS=n', 'DOC=n', 'EXTRA_CFLAGS="-Wno-error"', 'install', diff --git a/src/vos/README.md b/src/vos/README.md index 3db74d01366..db3c83c8229 100644 --- a/src/vos/README.md +++ b/src/vos/README.md @@ -58,7 +58,7 @@ Please refer to the Blob I/O (BIO) module for mor Special care is taken when developing and modifying the VOS layer because any software bug could corrupt data structures in persistent memory. The VOS, therefore, checksums its persistent data structures despite the presence of hardware ECC. -The VOS provides a lightweight I/O stack fully in user space, leveraging the PMDK open-source libraries developed to support this programming model. +The VOS provides a lightweight I/O stack fully in user space, leveraging the PMDK open-source libraries developed to support this programming model. @@ -73,7 +73,7 @@ It is worth noting that such transactions are different from the DAOS transactio Persistent memory transactions must guarantee consistency of VOS internal data structures when processing incoming requests, regardless of their epoch number. Transactions over persistent memory can be implemented in many different ways, e.g., undo logs, redo logs, a combination of both, or copy-on-write. -PMDK is an open source collection of libraries for using persistent memory, optimized specifically for NVRAM. +PMDK is an open source collection of libraries for using persistent memory, optimized specifically for NVRAM. Among these is the libpmemobj library, which implements relocatable persistent heaps called persistent memory pools. This includes memory allocation, transactions, and general facilities for persistent memory programming. The transactions are local to one thread (not multi-threaded) and rely on undo logs. diff --git a/src/vos/vos_dtx.c b/src/vos/vos_dtx.c index 4eefa622b7a..d4b3cbfa34d 100644 --- a/src/vos/vos_dtx.c +++ b/src/vos/vos_dtx.c @@ -10,7 +10,6 @@ */ #define D_LOGFAC DD_FAC(vos) -#include #include #include #include "vos_layout.h" diff --git a/utils/build.config b/utils/build.config index fb248a8ee23..6ab4ff9eae8 100644 --- a/utils/build.config +++ b/utils/build.config @@ -3,7 +3,7 @@ component=daos [commit_versions] ARGOBOTS = v1.1 -PMDK = 1.12.1 +PMDK = 2.0.0 ISAL = v2.30.0 ISAL_CRYPTO = v2.23.0 SPDK = v22.01.2 diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 124e7bc82f8..ae75e66951f 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -15,7 +15,7 @@ Name: daos Version: 2.5.100 -Release: 11%{?relval}%{?dist} +Release: 12%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -49,7 +49,7 @@ BuildRequires: libabt-devel >= 1.0rc1 BuildRequires: libjson-c-devel BuildRequires: boost-devel %endif -BuildRequires: libpmemobj-devel >= 1.12.1~rc1 +BuildRequires: libpmemobj-devel >= 2.0.0 %if (0%{?rhel} >= 8) BuildRequires: fuse3-devel >= 3 %else @@ -145,11 +145,11 @@ Requires: ndctl # needed to set PMem configuration goals in BIOS through control-plane %if (0%{?suse_version} >= 1500) Requires: ipmctl >= 03.00.00.0423 -Requires: libpmemobj1 >= 1.12.1~rc1-1.suse1500 +Requires: libpmemobj1 >= 2.0.0-1.suse1500 Requires: libfabric1 >= %{libfabric_version} %else Requires: ipmctl >= 03.00.00.0468 -Requires: libpmemobj >= 1.12.1~rc1-1%{?dist} +Requires: libpmemobj >= 2.0.0-1%{?dist} %endif Requires: libfabric >= %{libfabric_version} Requires: mercury >= %{mercury_version} @@ -585,6 +585,15 @@ getent passwd daos_agent >/dev/null || useradd -s /sbin/nologin -r -g daos_agent # No files in a shim package %changelog +* Fri Nov 17 2023 Tomasz Gromadzki 2.5.100-12 +- Update to PMDK 2.0.0 + * Remove libpmemblk from dependencies. + * Start using BUILD_EXAMPLES=n and BUILD_BENCHMARKS=n instead of patches. + * Stop using BUILD_RPMEM=n (removed) and NDCTL_DISABLE=y (invalid). + * Point https://github.com/pmem/pmdk as the main PMDK reference source. + NOTE: PMDK upgrade to 2.0.0 does not affect any API call used by DAOS. + libpmemobj (and libpmem) API stays unchanged. + * Wed Nov 15 2023 Jerome Soumagne 2.5.100-11 - Bump mercury min version to 2.3.1