Updating based on PR Feedback(5)

1. Added new module parameter zfs_dio_enabled which allows for all reads
   and writes to pass through the ARC. This module parameter can be set
   to 0 by default in OpenZFS 2.3 release if necessary.
2. Updated ZTS direct tests to account for the new zfs_dio_enabled
   module parameter.
3. Updated libzfs.abi to account for changes.
4. The recently merged commit 82ff9aa added a pretty printer for ZIO
   pipeline stages and zio flags. I updated zfs_valstr.c to account for
   the added ZIO pipeline stage ZIO_STAGE_DIO_CHECKSUM_VERIFY as well as
   the zio flag ZIO_FLAG_DIO_CHKSUM_ERR that we added in the Direct I/O
   code.

Signed-off-by: Brian Atkinson <[email protected]>

man/man4/zfs.4

@@ -291,6 +291,14 @@ Default dnode block size as a power of 2.
 .It Sy zfs_default_ibs Ns = Ns Sy 17 Po 128 KiB Pc Pq int
 Default dnode indirect block size as a power of 2.
 .
+.It Sy zfs_dio_enabled Ns = Ns Sy 0 Ns | Ns 1 Pq int
+Enable Direct I/O.
+If this setting is 0, then all I/O requests will be directed through the ARC
+acting as though the dataset property
+.Sy direct
+was set to
+.Sy disabled .
+.
 .It Sy zfs_history_output_max Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64
 When attempting to log an output nvlist of an ioctl in the on-disk history,
 the output will not be stored if it is larger than this size (in bytes).

module/zcommon/zfs_valstr.c

@@ -218,6 +218,7 @@ _VALSTR_BITFIELD_IMPL(zio_flag,
 	{ '.', "NP", "NOPWRITE" },
 	{ '.', "EX", "REEXECUTED" },
 	{ '.', "DG", "DELEGATED" },
+	{ '.', "DC", "DIO_CHKSUM_ERR" },
 )
 /* END CSTYLED */
 
@@ -252,6 +253,7 @@ _VALSTR_BITFIELD_IMPL(zio_stage,
 	{ 'V', "VD", "VDEV_IO_DONE" },
 	{ 'V', "VA", "VDEV_IO_ASSESS" },
 	{ 'C', "CV", "CHECKSUM_VERIFY" },
+	{ 'C', "DC", "DIO_CHECKSUM_VERIFY" },
 	{ 'X', "X ", "DONE" },
 )
 /* END CSTYLED */

module/zfs/zfs_vnops.c

@@ -74,6 +74,14 @@ int zfs_bclone_enabled = 1;
  */
 static int zfs_bclone_wait_dirty = 0;
 
+/*
+ * Enable Direct I/O. If this setting is 0, then all I/O requests will be
+ * directed through the ARC acting as though the dataset property direct was
+ * set to disabled.
+ */
+static int zfs_dio_enabled = 1;
+
+
 /*
  * Maximum bytes to read per chunk in zfs_read().
  */
@@ -227,7 +235,7 @@ zfs_setup_direct(struct znode *zp, zfs_uio_t *uio, zfs_uio_rw_t rw,
 	int ioflag = *ioflagp;
 	int error = 0;
 
-	if (os->os_direct == ZFS_DIRECT_DISABLED ||
+	if (!zfs_dio_enabled || os->os_direct == ZFS_DIRECT_DISABLED ||
 	    zn_has_cached_data(zp, zfs_uio_offset(uio),
 	    zfs_uio_offset(uio) + zfs_uio_resid(uio) - 1)) {
 		/*
@@ -1805,3 +1813,6 @@ ZFS_MODULE_PARAM(zfs, zfs_, bclone_enabled, INT, ZMOD_RW,
 
 ZFS_MODULE_PARAM(zfs, zfs_, bclone_wait_dirty, INT, ZMOD_RW,
 	"Wait for dirty blocks when cloning");
+
+ZFS_MODULE_PARAM(zfs, zfs_, dio_enabled, INT, ZMOD_RW,
+	"Enable Direct I/O");

tests/zfs-tests/include/tunables.cfg

@@ -101,6 +101,7 @@ VOL_RECURSIVE			vol.recursive			UNSUPPORTED
 VOL_USE_BLK_MQ			UNSUPPORTED			zvol_use_blk_mq
 BCLONE_ENABLED			bclone_enabled			zfs_bclone_enabled
 BCLONE_WAIT_DIRTY		bclone_wait_dirty		zfs_bclone_wait_dirty
+DIO_ENABLED			dio_enabled			zfs_dio_enabled
 XATTR_COMPAT			xattr_compat			zfs_xattr_compat
 ZEVENT_LEN_MAX			zevent.len_max			zfs_zevent_len_max
 ZEVENT_RETAIN_MAX		zevent.retain_max		zfs_zevent_retain_max

tests/zfs-tests/tests/functional/direct/cleanup.ksh

@@ -28,4 +28,10 @@
 
 verify_runnable "global"
 
-default_cleanup
+default_cleanup_noexit
+
+if tunable_exists DIO_ENABLED ; then
+	log_must restore_tunable DIO_ENABLED
+fi
+
+log_pass

tests/zfs-tests/tests/functional/direct/setup.ksh

@@ -27,6 +27,11 @@
 . $STF_SUITE/include/libtest.shlib
 verify_runnable "global"
 
+if tunable_exists DIO_ENABLED ; then
+	log_must save_tunable DIO_ENABLED
+	log_must set_tunable32 DIO_ENABLED 1
+fi
+
 default_raidz_setup_noexit "$DISKS"
 log_must zfs set compression=off $TESTPOOL/$TESTFS
 log_pass