Revert "Adjust prefetch parameters."

This reverts commit 81be809.

This patch was identified by HPE as significantly hurting performance
on all-flash pools.

The reason for the regression and best solution isn't yet determined,
and so this is only a local patch while that is investigated.

include/sys/dmu_zfetch.h

@@ -36,6 +36,8 @@
 extern "C" {
 #endif
 
+extern uint64_t	zfetch_array_rd_sz;
+
 struct dnode;				/* so we can reference dnode */
 
 typedef struct zfetch {

man/man4/zfs.4

@@ -547,6 +547,8 @@ A micro ZAP is upgraded to a fat ZAP, once it grows beyond the specified size.
 .It Sy zfetch_hole_shift Ns = Ns Sy 2 Pq uint
 Log2 fraction of holes in speculative prefetch stream allowed for it to
 proceed.
+.It Sy zfetch_array_rd_sz Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64
+If prefetching is enabled, disable prefetching for reads larger than this size.
 .
 .It Sy zfetch_min_distance Ns = Ns Sy 4194304 Ns B Po 4 MiB Pc Pq uint
 Min bytes to prefetch per stream.

module/zfs/dmu.c

@@ -89,11 +89,7 @@ static int zfs_dmu_offset_next_sync = 1;
  * helps to limit the amount of memory that can be used by prefetching.
  * Larger objects should be prefetched a bit at a time.
  */
-#ifdef _ILP32
-uint_t dmu_prefetch_max = 8 * 1024 * 1024;
-#else
 uint_t dmu_prefetch_max = 8 * SPA_MAXBLOCKSIZE;
-#endif
 
 const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES] = {
 	{DMU_BSWAP_UINT8,  TRUE,  FALSE, FALSE, "unallocated"		},
@@ -557,7 +553,8 @@ dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
 		zio = zio_root(dn->dn_objset->os_spa, NULL, NULL,
 		    ZIO_FLAG_CANFAIL);
 	blkid = dbuf_whichblock(dn, 0, offset);
-	if ((flags & DMU_READ_NO_PREFETCH) == 0) {
+	if ((flags & DMU_READ_NO_PREFETCH) == 0 &&
+	    length <= zfetch_array_rd_sz) {
 		/*
 		 * Prepare the zfetch before initiating the demand reads, so
 		 * that if multiple threads block on same indirect block, we

module/zfs/dmu_zfetch.c

@@ -52,23 +52,18 @@ static unsigned int	zfetch_max_streams = 8;
 static unsigned int	zfetch_min_sec_reap = 1;
 /* max time before stream delete */
 static unsigned int	zfetch_max_sec_reap = 2;
-#ifdef _ILP32
-/* min bytes to prefetch per stream (default 2MB) */
-static unsigned int	zfetch_min_distance = 2 * 1024 * 1024;
-/* max bytes to prefetch per stream (default 8MB) */
-unsigned int	zfetch_max_distance = 8 * 1024 * 1024;
-#else
 /* min bytes to prefetch per stream (default 4MB) */
 static unsigned int	zfetch_min_distance = 4 * 1024 * 1024;
 /* max bytes to prefetch per stream (default 64MB) */
 unsigned int	zfetch_max_distance = 64 * 1024 * 1024;
-#endif
 /* max bytes to prefetch indirects for per stream (default 64MB) */
 unsigned int	zfetch_max_idistance = 64 * 1024 * 1024;
 /* max request reorder distance within a stream (default 16MB) */
 unsigned int	zfetch_max_reorder = 16 * 1024 * 1024;
 /* Max log2 fraction of holes in a stream */
 unsigned int	zfetch_hole_shift = 2;
+/* max number of bytes in an array_read in which we allow prefetching (1MB) */
+uint64_t	zfetch_array_rd_sz = 1024 * 1024;
 
 typedef struct zfetch_stats {
 	kstat_named_t zfetchstat_hits;
@@ -771,3 +766,6 @@ ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, max_reorder, UINT, ZMOD_RW,
 
 ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, hole_shift, UINT, ZMOD_RW,
 	"Max log2 fraction of holes in a stream");
+
+ZFS_MODULE_PARAM(zfs_prefetch, zfetch_, array_rd_sz, U64, ZMOD_RW,
+	"Number of bytes in a array_read");