From a423ea360642658cf12d4fa9f026e2224b020441 Mon Sep 17 00:00:00 2001 From: "Olaf P. Faaland" Date: Thu, 2 May 2024 17:14:07 -0700 Subject: [PATCH] Revert "Adjust prefetch parameters." This reverts commit 81be809a25b8d3ef859f203664d1981163d87983. This patch was identified by HPE as significantly hurting performance on all-flash pools. The reason for the regression and best solution isn't yet determined, and so this is only a local patch while that is investigated. --- include/sys/dmu_zfetch.h | 2 ++ man/man4/zfs.4 | 2 ++ module/zfs/dmu.c | 7 ++----- module/zfs/dmu_zfetch.c | 12 +++++------- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/include/sys/dmu_zfetch.h b/include/sys/dmu_zfetch.h index 322472fb1ae2..cc676c01730b 100644 --- a/include/sys/dmu_zfetch.h +++ b/include/sys/dmu_zfetch.h @@ -36,6 +36,8 @@ extern "C" { #endif +extern uint64_t zfetch_array_rd_sz; + struct dnode; /* so we can reference dnode */ typedef struct zfetch { diff --git a/man/man4/zfs.4 b/man/man4/zfs.4 index 1191cc962492..a69ebd19d74f 100644 --- a/man/man4/zfs.4 +++ b/man/man4/zfs.4 @@ -547,6 +547,8 @@ A micro ZAP is upgraded to a fat ZAP, once it grows beyond the specified size. .It Sy zfetch_hole_shift Ns = Ns Sy 2 Pq uint Log2 fraction of holes in speculative prefetch stream allowed for it to proceed. +.It Sy zfetch_array_rd_sz Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64 +If prefetching is enabled, disable prefetching for reads larger than this size. . .It Sy zfetch_min_distance Ns = Ns Sy 4194304 Ns B Po 4 MiB Pc Pq uint Min bytes to prefetch per stream. diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c index d8d5cfdbd230..4fd3c6df384b 100644 --- a/module/zfs/dmu.c +++ b/module/zfs/dmu.c @@ -89,11 +89,7 @@ static int zfs_dmu_offset_next_sync = 1; * helps to limit the amount of memory that can be used by prefetching. * Larger objects should be prefetched a bit at a time. */ -#ifdef _ILP32 -uint_t dmu_prefetch_max = 8 * 1024 * 1024; -#else uint_t dmu_prefetch_max = 8 * SPA_MAXBLOCKSIZE; -#endif const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES] = { {DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "unallocated" }, @@ -557,7 +553,8 @@ dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length, zio = zio_root(dn->dn_objset->os_spa, NULL, NULL, ZIO_FLAG_CANFAIL); blkid = dbuf_whichblock(dn, 0, offset); - if ((flags & DMU_READ_NO_PREFETCH) == 0) { + if ((flags & DMU_READ_NO_PREFETCH) == 0 && + length <= zfetch_array_rd_sz) { /* * Prepare the zfetch before initiating the demand reads, so * that if multiple threads block on same indirect block, we diff --git a/module/zfs/dmu_zfetch.c b/module/zfs/dmu_zfetch.c index ed50f1889b59..5c302d1d00c4 100644 --- a/module/zfs/dmu_zfetch.c +++ b/module/zfs/dmu_zfetch.c @@ -52,23 +52,18 @@ static unsigned int zfetch_max_streams = 8; static unsigned int zfetch_min_sec_reap = 1; /* max time before stream delete */ static unsigned int zfetch_max_sec_reap = 2; -#ifdef _ILP32 -/* min bytes to prefetch per stream (default 2MB) */ -static unsigned int zfetch_min_distance = 2 * 1024 * 1024; -/* max bytes to prefetch per stream (default 8MB) */ -unsigned int zfetch_max_distance = 8 * 1024 * 1024; -#else /* min bytes to prefetch per stream (default 4MB) */ static unsigned int zfetch_min_distance = 4 * 1024 * 1024; /* max bytes to prefetch per stream (default 64MB) */ unsigned int zfetch_max_distance = 64 * 1024 * 1024; -#endif /* max bytes to prefetch indirects for per stream (default 64MB) */ unsigned int zfetch_max_idistance = 64 * 1024 * 1024; /* max request reorder distance within a stream (default 16MB) */ unsigned int zfetch_max_reorder = 16 * 1024 * 1024; /* Max log2 fraction of holes in a stream */ unsigned int zfetch_hole_shift = 2; +/* max number of bytes in an array_read in which we allow prefetching (1MB) */ +uint64_t zfetch_array_rd_sz = 1024 * 1024; typedef struct zfetch_stats { kstat_named_t zfetchstat_hits; @@ -771,3 +766,6 @@ ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, max_reorder, UINT, ZMOD_RW, ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, hole_shift, UINT, ZMOD_RW, "Max log2 fraction of holes in a stream"); + +ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, array_rd_sz, U64, ZMOD_RW, + "Number of bytes in a array_read");