Add zpool events tests

* events_001_pos - Verify the expected events are generated when
  invoking the various zpool sub-commands.  These events must
  appear in `zpool event` and be consumed by the ZED.

* events_002_pos - Verify the ZED consumes events which were
  generated while it wasn't running when it is started.
  Additionally, verify that events are only processed once.

As part of this change the default.cfg used by the test suite
was changed to a default.cfg.in file.  This was needed so the
install location of all zed scripts, not only the enabled ones,
could be reliably determined.

Signed-off-by: Brian Behlendorf <[email protected]>
Closes #6128

configure.ac

@@ -168,6 +168,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/cmd/threadsappend/Makefile
 	tests/zfs-tests/cmd/xattrtest/Makefile
 	tests/zfs-tests/include/Makefile
+	tests/zfs-tests/include/default.cfg
 	tests/zfs-tests/tests/Makefile
 	tests/zfs-tests/tests/functional/Makefile
 	tests/zfs-tests/tests/functional/acl/Makefile
@@ -234,6 +235,7 @@ AC_CONFIG_FILES([
 	tests/zfs-tests/tests/functional/ctime/Makefile
 	tests/zfs-tests/tests/functional/delegate/Makefile
 	tests/zfs-tests/tests/functional/devices/Makefile
+	tests/zfs-tests/tests/functional/events/Makefile
 	tests/zfs-tests/tests/functional/exec/Makefile
 	tests/zfs-tests/tests/functional/fault/Makefile
 	tests/zfs-tests/tests/functional/features/async_destroy/Makefile

scripts/zfs-tests.sh

@@ -401,6 +401,8 @@ constrain_path
 # Check if ksh exists
 #
 [ -e "$STF_PATH/ksh" ] || fail "This test suite requires ksh."
+[ -e "$STF_SUITE/include/default.cfg" ] || fail \
+    "Missing $STF_SUITE/include/default.cfg file."
 
 #
 # Verify the ZFS module stack if loaded.

tests/runfiles/linux.run

@@ -344,6 +344,9 @@ tests = ['zfs_allow_001_pos', 'zfs_allow_002_pos',
 [tests/functional/devices]
 tests = ['devices_001_pos', 'devices_002_neg', 'devices_003_pos']
 
+[tests/functional/events]
+tests = ['events_001_pos', 'events_002_pos']
+
 [tests/functional/exec]
 tests = ['exec_001_pos', 'exec_002_neg']
 

tests/zfs-tests/include/.gitignore

@@ -0,0 +1 @@
+/default.cfg

tests/zfs-tests/include/Makefile.am

@@ -6,3 +6,8 @@ dist_pkgdata_SCRIPTS = \
 	math.shlib \
 	properties.shlib \
 	zpool_script.shlib
+
+EXTRA_DIST=default.cfg.in
+
+distclean-local::
+	-$(RM) $(dist_pkgdata_SCRIPTS)

tests/zfs-tests/include/default.cfg → tests/zfs-tests/include/default.cfg.in

@@ -33,8 +33,11 @@
 . $STF_SUITE/include/libtest.shlib
 
 # ZFS Directories
-export ZEDLETDIR=${ZEDLETDIR:-/etc/zfs/zed.d}
-export ZPOOLSCRIPTDIR=${ZPOOLSCRIPTDIR:-/etc/zfs/zpool.d}
+export prefix=@prefix@
+export exec_prefix=@exec_prefix@
+export ZEDLET_ETC_DIR=${ZEDLET_ETC_DIR:-@sysconfdir@/zfs/zed.d}
+export ZEDLET_LIBEXEC_DIR=${ZEDLET_LIBEXEC_DIR:-@libexecdir@/zfs/zed.d}
+export ZPOOL_SCRIPT_DIR=${ZPOOL_SCRIPT_DIR:-@sysconfdir@/zfs/zpool.d}
 
 # Define run length constants
 export RT_LONG="3"

tests/zfs-tests/include/libtest.shlib

@@ -3280,11 +3280,16 @@ function zed_setup
 
 	# Setup minimal ZED configuration.  Individual test cases should
 	# add additional ZEDLETs as needed for their specific test.
-	log_must cp ${ZEDLETDIR}/zed.rc $ZEDLET_DIR
-	log_must cp ${ZEDLETDIR}/zed-functions.sh $ZEDLET_DIR
-	log_must cp ${ZEDLETDIR}/all-syslog.sh $ZEDLET_DIR
+	log_must cp ${ZEDLET_ETC_DIR}/zed.rc $ZEDLET_DIR
+	log_must cp ${ZEDLET_ETC_DIR}/zed-functions.sh $ZEDLET_DIR
 
-	log_must zpool events -c
+	# Customize the zed.rc file to enable the full debug log.
+	log_must sed -i '/\#ZED_DEBUG_LOG=.*/d' $ZEDLET_DIR/zed.rc
+	echo "ZED_DEBUG_LOG=$ZEDLET_DIR/zed.debug.log" >>$ZEDLET_DIR/zed.rc
+
+	log_must cp ${ZEDLET_LIBEXEC_DIR}/all-syslog.sh $ZEDLET_DIR
+	log_must cp ${ZEDLET_LIBEXEC_DIR}/all-debug.sh $ZEDLET_DIR
+	log_must touch $ZEDLET_DIR/zed.debug.log
 }
 
 #
@@ -3299,8 +3304,10 @@ function zed_cleanup
 	log_must rm -f ${ZEDLET_DIR}/zed.rc
 	log_must rm -f ${ZEDLET_DIR}/zed-functions.sh
 	log_must rm -f ${ZEDLET_DIR}/all-syslog.sh
+	log_must rm -f ${ZEDLET_DIR}/all-debug.sh
 	log_must rm -f ${ZEDLET_DIR}/zed.pid
 	log_must rm -f ${ZEDLET_DIR}/zedlog
+	log_must rm -f ${ZEDLET_DIR}/zed.debug.log
 	log_must rm -f ${ZEDLET_DIR}/state
 	log_must rm -f $VDEVID_CONF_ETC
 	log_must rm -f $VDEVID_CONF
@@ -3332,6 +3339,8 @@ function zed_start
 	# output to zedlog
 	log_must eval "zed -vF -d $ZEDLET_DIR -p $ZEDLET_DIR/zed.pid" \
 	    "-s $ZEDLET_DIR/state 2>${ZEDLET_DIR}/zedlog &"
+
+	return 0
 }
 
 #
@@ -3343,10 +3352,13 @@ function zed_stop
 		return
 	fi
 
+	log_note "Stopping ZED"
 	if [[ -f ${ZEDLET_DIR}/zed.pid ]]; then
 		zedpid=$(cat ${ZEDLET_DIR}/zed.pid)
 		log_must kill $zedpid
 	fi
+
+	return 0
 }
 
 #

tests/zfs-tests/tests/functional/Makefile.am

@@ -14,6 +14,7 @@ SUBDIRS = \
 	ctime \
 	delegate \
 	devices \
+	events \
 	exec \
 	fault \
 	features \

tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_003_pos.ksh

@@ -53,10 +53,10 @@ else
 	testpool="${TESTPOOL%%/*}"
 fi
 
-files="$(ls $ZPOOLSCRIPTDIR)"
+files="$(ls $ZPOOL_SCRIPT_DIR)"
 scripts=""
 for i in $files ; do
-	if [ ! -x "$ZPOOLSCRIPTDIR/$i" ] ; then
+	if [ ! -x "$ZPOOL_SCRIPT_DIR/$i" ] ; then
 		# Skip non-executables
 		continue
 	fi

tests/zfs-tests/tests/functional/cli_user/zpool_iostat/zpool_iostat_005_pos.ksh

@@ -53,10 +53,10 @@ else
 	testpool=${TESTPOOL%%/*}
 fi
 
-files="$(ls $ZPOOLSCRIPTDIR)"
+files="$(ls $ZPOOL_SCRIPT_DIR)"
 scripts=""
 for i in $files ; do
-	if [ ! -x "$ZPOOLSCRIPTDIR/$i" ] ; then
+	if [ ! -x "$ZPOOL_SCRIPT_DIR/$i" ] ; then
 		# Skip non-executables
 		continue
 	fi

tests/zfs-tests/tests/functional/events/Makefile.am

@@ -0,0 +1,8 @@
+pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/events
+dist_pkgdata_SCRIPTS = \
+	setup.ksh \
+	cleanup.ksh \
+	events.cfg \
+	events_common.kshlib \
+	events_001_pos.ksh \
+	events_002_pos.ksh

tests/zfs-tests/tests/functional/events/cleanup.ksh

@@ -0,0 +1,31 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+zed_cleanup
+
+default_cleanup

tests/zfs-tests/tests/functional/events/events.cfg

@@ -0,0 +1,38 @@
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+export MPOOL=mpool.$$
+
+VDEV1=$TEST_BASE_DIR/vdev1
+VDEV2=$TEST_BASE_DIR/vdev2
+VDEV3=$TEST_BASE_DIR/vdev3
+VDEV4=$TEST_BASE_DIR/vdev4
+
+export TMP_EVENTS=/tmp/tmp_events.$$
+export TMP_EVENTS_FULL=/tmp/tmp_events_full.$$
+export TMP_EVENT_FULL=/tmp/tmp_event_full.$$
+export TMP_EVENTS_ZED=/tmp/tmp_events_zed.$$
+export TMP_EVENT_ZED=/tmp/tmp_event_zed.$$

tests/zfs-tests/tests/functional/events/events_001_pos.ksh

@@ -0,0 +1,134 @@
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# The contents of this file are subject to the terms of the
+# Common Development and Distribution License (the "License").
+# You may not use this file except in compliance with the License.
+#
+# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+# or http://www.opensolaris.org/os/licensing.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# When distributing Covered Code, include this CDDL HEADER in each
+# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+# If applicable, add the following below this CDDL HEADER, with the
+# fields enclosed by brackets "[]" replaced with your own identifying
+# information: Portions Copyright [yyyy] [name of copyright owner]
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2017 by Lawrence Livermore National Security, LLC.
+# Use is subject to license terms.
+#
+
+# DESCRIPTION:
+# Verify zpool events command logs events.
+#
+# STRATEGY:
+# 1. Execute zpool sub-commands on a pool.
+# 2. Verify the expected events are logged in 'zpool events'.
+# 3. Verify the expected events are logged by the ZED.
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/events/events_common.kshlib
+
+verify_runnable "both"
+
+function cleanup
+{
+	if poolexists $MPOOL; then
+		destroy_pool $MPOOL
+	fi
+
+	for file in $VDEV1 $VDEV2 $VDEV3 $VDEV4; do
+		[[ -f $file ]] && rm -f $file
+	done
+
+	log_must zed_stop
+}
+
+log_assert "Verify zpool sub-commands generate expected events"
+log_onexit cleanup
+
+log_must truncate -s $MINVDEVSIZE $VDEV1 $VDEV2 $VDEV3 $VDEV4
+log_must zed_start
+
+# Create a mirrored pool with two devices.
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.config_sync" \
+    -e "sysevent.fs.zfs.pool_create" \
+    "zpool create $MPOOL mirror $VDEV1 $VDEV2"
+
+# Add a cache device then remove it.
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.config_sync" \
+    -e "sysevent.fs.zfs.vdev_add" \
+    "zpool add -f $MPOOL spare $VDEV3"
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.vdev_remove_aux" \
+    "zpool remove $MPOOL $VDEV3"
+
+# Add a log device then remove it.
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.config_sync" \
+    -e "sysevent.fs.zfs.vdev_add" \
+    "zpool add -f $MPOOL log $VDEV3"
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.vdev_remove_dev" \
+    "zpool remove $MPOOL $VDEV3"
+
+# Offline then online a device.
+run_and_verify -p "$MPOOL"\
+    -e "resource.fs.zfs.statechange" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool offline $MPOOL $VDEV1"
+run_and_verify -p "$MPOOL" \
+    -e "resource.fs.zfs.statechange" \
+    -e "sysevent.fs.zfs.vdev_online" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool online $MPOOL $VDEV1"
+
+# Attach then detach a device from the mirror.
+run_and_verify -p "$MPOOL" \
+     -e "sysevent.fs.zfs.vdev_attach" \
+    "zpool attach $MPOOL $VDEV1 $VDEV4"
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.vdev_remove" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool detach $MPOOL $VDEV4"
+
+# Replace a device
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.vdev_attach" \
+    -e "sysevent.fs.zfs.resilver_start" \
+    -e "sysevent.fs.zfs.resilver_finish" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool replace -f $MPOOL $VDEV1 $VDEV4"
+
+# Scrub a pool.
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.scrub_start" \
+    -e "sysevent.fs.zfs.scrub_finish" \
+    "zpool scrub $MPOOL"
+
+# Export then import a pool (may change to a pool_export event)
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.pool_destroy" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool export $MPOOL"
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.pool_import" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool import -d $TEST_BASE_DIR $MPOOL"
+
+# Destroy the pool
+run_and_verify -p "$MPOOL" \
+    -e "sysevent.fs.zfs.pool_destroy" \
+    -e "sysevent.fs.zfs.config_sync" \
+    "zpool destroy $MPOOL"
+
+log_pass "Verify zpool sub-commands generate expected events"