QA-35139 zcache and slog device sharing azure (openzfs#519)

Use zdb lib function to be compatible with object store for all supported tests.
*  pool checkpoint tests
* zdb

tests/zfs-tests/include/object_store.shlib

@@ -243,16 +243,29 @@ function get_zoa_debug_log
 # Takes a list of devices and configures it to be
 # used a zettacache
 #
-function configure_zettacache
+function zcache_add_devices
 {
-	typeset zcache_devices=$(echo $@ | tr ' ' ',')
-	stop_zfs_object_agent
-	# Truncate the zoa log to verify cache information
+	typeset zcache_devices="$@"
 	sudo truncate -s 0 $(get_zoa_debug_log)
-	for cache_dev in ${ZETTACACHE_DEVICES}; do
-		invalidate_zcache_dev $cache_dev
+	for cache_dev in $zcache_devices; do
+		zcache add $cache_dev
 	done
-	start_zfs_object_agent
+}
+
+#
+# Helper function to invalidate a list
+# of cache devices.
+# The invalidation is done by removing the
+# associated label.
+#
+function zcache_remove_devices
+{
+	typeset cache_devices="$@"
+	sudo systemctl stop zfs-object-agent
+	for cache_dev in $cache_devices; do
+		zcache labelclear -f "$cache_dev"
+	done
+	sudo systemctl start zfs-object-agent
 }
 
 #
@@ -269,7 +282,7 @@ function verify_zcache_added_from_logfile
 	# So we retry for 10 time sleeping for 1 seconds
 	# after each retry
 	while [ $retry_count -lt $max_retries ]; do
-		grep -q "opening cache file ${cache_device}:" $(get_zoa_debug_log)
+		grep -q "opening cache file \"${cache_device}\":" $(get_zoa_debug_log)
 		[ $? -eq 0 ] && return 0
 		sleep 1
 		retry_count=$((retry_count+1))

tests/zfs-tests/tests/functional/cli_root/zdb/zdb_006_pos.ksh

@@ -46,10 +46,10 @@ verify_runnable "global"
 default_setup_noexit "$DISKS"
 log_must zfs snap $TESTPOOL/$TESTFS@snap
 
-log_must zdb -d $TESTPOOL
-log_must zdb -d $TESTPOOL/
-log_must zdb -d $TESTPOOL/$TESTFS
-log_must zdb -d $TESTPOOL/$TESTFS@snap
+log_must run_zdb -e "-d" -p $TESTPOOL
+log_must run_zdb -e "-d" -p $TESTPOOL/
+log_must run_zdb -e "-d" -p $TESTPOOL/$TESTFS
+log_must run_zdb -e "-d" -p $TESTPOOL/$TESTFS@snap
 
 log_must zpool export $TESTPOOL
 

tests/zfs-tests/tests/functional/cli_root/zdb/zdb_args_neg.ksh

@@ -58,7 +58,7 @@ set -A args "create" "add" "destroy" "import fakepool" \
     "setvprop" "blah blah" "-%" "--?" "-*" "-=" \
     "-j" "-n" "-o" "-p" "-p /tmp" \
     "-t" "-w" "-E" "-H" "-I" "-J" "-K" \
-    "-Q" "-R" "-T" "-W"
+    "-Q" "-R" "-d -T" "-W"
 
 log_assert "Execute zdb using invalid parameters."
 

tests/zfs-tests/tests/functional/cli_root/zdb/zdb_args_pos.ksh

@@ -59,7 +59,7 @@ function test_imported_pool
 {
 	typeset -a args=("-A" "-b" "-C" "-c" "-d" "-D" "-G" "-h" "-i" "-L" \
             "-M" "-P" "-s" "-v" "-Y" "-y")
-        for i in ${args[@]}; do
+	for i in ${args[@]}; do
 		log_must eval "run_zdb -e '$i' -p $TESTPOOL >/dev/null"
 	done
 }
@@ -69,7 +69,7 @@ function test_exported_pool
 	log_must zpool export $TESTPOOL
 	typeset -a args=("-A" "-b" "-C" "-c" "-d" "-D" "-F" "-G" "-h" "-i" "-L" "-M" \
             "-P" "-s" "-v" "-X" "-Y" "-y")
-        for i in ${args[@]}; do
+	for i in ${args[@]}; do
 		log_must eval "run_zdb -e '-e $i' -p $TESTPOOL >/dev/null"
 	done
 	log_must import_pool -p $TESTPOOL

tests/zfs-tests/tests/functional/cli_root/zdb/zdb_checksum.ksh

@@ -51,13 +51,13 @@ obj=${array[0]}
 log_note "file $init_data has object number $obj"
 sync_pool $TESTPOOL
 
-output=$(zdb -ddddddbbbbbb $TESTPOOL/$TESTFS $obj 2> /dev/null \
+output=$(run_zdb -e "-ddddddbbbbbb" -p "$TESTPOOL/$TESTFS $obj" 2> /dev/null \
     | grep -m 1 "L0 DVA")
 dva=$(sed -Ene 's/^.+DVA\[0\]=<([^>]+)>.*$/\1/p' <<< "$output")
 log_note "block 0 of $init_data has a DVA of $dva"
 cksum_expected=$(sed -Ene 's/^.+ cksum=([a-z0-9:]+)$/\1/p' <<< "$output")
 log_note "expecting cksum $cksum_expected"
-output=$(zdb -R $TESTPOOL $dva:c 2> /dev/null)
+output=$(run_zdb -e "-R" -p "$TESTPOOL $dva:c" 2> /dev/null)
 grep -q $cksum_expected <<<"$output" ||
 	log_fail "zdb -R failed to print the correct checksum"
 

tests/zfs-tests/tests/functional/cli_root/zdb/zdb_decompress.ksh

@@ -71,7 +71,7 @@ set -A array $listing
 obj=${array[0]}
 log_note "file $init_data has object number $obj"
 
-output=$(zdb -ddddddbbbbbb $TESTPOOL/$TESTFS $obj 2> /dev/null \
+output=$(run_zdb -e "-ddddddbbbbbb" -p "$TESTPOOL/$TESTFS $obj" 2> /dev/null \
     |grep -m 1 "L0 DVA")
 dva=$(sed -Ene 's/^.+DVA\[0\]=<([^>]+)>.*$/\1/p' <<< "$output")
 log_note "block 0 of $init_data has a DVA of $dva"
@@ -81,20 +81,20 @@ size_str=$(sed -Ene 's/^.+ size=([^ ]+) .*$/\1/p' <<< "$output")
 log_note "block size $size_str"
 
 IFS=: read -r vdev offset _ <<< "$dva"
-output=$(zdb -R $TESTPOOL $vdev:$offset:$size_str:d)
+output=$(zdb -e "-R" -p "$TESTPOOL $vdev:$offset:$size_str:d")
 echo $output | grep -q $pattern || log_fail "zdb -R :d failed to decompress the data properly"
 
-output=$(zdb -R $TESTPOOL $vdev:$offset:$size_str:dr)
+output=$(run_zdb -e "-R" -p "$TESTPOOL $vdev:$offset:$size_str:dr")
 echo $output | grep -q $four_k || log_fail "zdb -R :dr failed to decompress the data properly"
 
-output=$(zdb -R $TESTPOOL $vdev:$offset:$size_str:dr)
+output=$(run_zdb -e "-R" -p "$TESTPOOL $vdev:$offset:$size_str:dr")
 result=${#output}
 (( $result != $blksize)) && log_fail \
 "zdb -R failed to decompress the data to the length (${#output} != $size_str)"
 
 # decompress using lsize
 IFS=/ read -r lsize psize _ <<< "$size_str"
-output=$(zdb -R $TESTPOOL $vdev:$offset:$lsize:dr)
+output=$(run_zdb -e "-R" -p "$TESTPOOL $vdev:$offset:$lsize:dr")
 result=${#output}
 (( $result != $blksize)) && log_fail \
 "zdb -R failed to decompress the data (length ${#output} != $blksize)"

tests/zfs-tests/tests/functional/cli_root/zdb/zdb_decompress_zstd.ksh

@@ -65,7 +65,7 @@ sync_pool $TESTPOOL true
 read -r obj _ < <(ls -i $init_data)
 log_note "file $init_data has object number $obj"
 
-output=$(zdb -Zddddddbbbbbb $TESTPOOL/$TESTFS $obj 2> /dev/null \
+output=$(run_zdb -e "-Zddddddbbbbbb" -p "$TESTPOOL/$TESTFS $obj" 2> /dev/null \
     | grep -m 1 "L0 DVA")
 dva=$(sed -Ene 's/^.+DVA\[0\]=<([^>]+)>.*$/\1/p' <<< "$output")
 log_note "block 0 of $init_data has a DVA of $dva"
@@ -96,7 +96,7 @@ log_note "ZSTD level $zstd_level"
 
 IFS=':' read -r vdev offset _ <<<"$dva"
 # Check the first 1024 bytes
-output=$(ZDB_NO_ZLE="true" zdb -R $TESTPOOL $vdev:$offset:$size_str:dr 2> /dev/null)
+output=$(ZDB_NO_ZLE="true" run_zdb -e "-R" -p "$TESTPOOL $vdev:$offset:$size_str:dr" 2> /dev/null)
 (( ${#output} + 1 != $blksize )) && log_fail \
 "zdb -Z failed to decompress the data to the expected length (${#output} != $lsize_bytes)"
 cmp $init_data - <<< "$output" ||

tests/zfs-tests/tests/functional/cli_root/zdb/zdb_display_block.ksh

@@ -72,37 +72,37 @@ set -A array $listing
 obj=${array[0]}
 log_note "file $init_data has object number $obj"
 
-output=$(zdb -ddddddbbbbbb $TESTPOOL/$TESTFS $obj 2> /dev/null \
+output=$(run_zdb -e "-ddddddbbbbbb" -p "$TESTPOOL/$TESTFS $obj" 2> /dev/null \
     |grep -m 1 "L1  DVA" )
 dva=$(sed -Ene 's/^.+DVA\[0\]=<([^>]+)>.*/\1/p' <<< "$output")
 log_note "first L1 block $init_data has a DVA of $dva"
-output=$(zdb -ddddddbbbbbb $TESTPOOL/$TESTFS $obj 2> /dev/null \
+output=$(run_zdb -e "-ddddddbbbbbb" -p "$TESTPOOL/$TESTFS $obj" 2> /dev/null \
     |grep -m 1 "L0 DVA" )
 blk_out0=${output##*>}
 blk_out0=${blk_out0##+([[:space:]])}
 
-output=$(zdb -ddddddbbbbbb $TESTPOOL/$TESTFS $obj 2> /dev/null \
+output=$(run_zdb -e "-ddddddbbbbbb" -p "$TESTPOOL/$TESTFS $obj" 2> /dev/null \
     |grep -m 1 "1000  L0 DVA" )
 blk_out1=${output##*>}
 blk_out1=${blk_out1##+([[:space:]])}
 
-output=$(export ZDB_NO_ZLE=\"true\"; zdb -R $TESTPOOL $dva:bd\
+output=$(export ZDB_NO_ZLE=\"true\"; run_zdb -e "-R" -p "$TESTPOOL $dva:bd"\
     2> /dev/null)
 output=${output##*>}
 output=${output##+([[:space:]])}
 if [ "$output" != "$blk_out0" ]; then
 	log_fail "zdb -R :bd (block 0 display/decompress) failed"
 fi
 
-output=$(export ZDB_NO_ZLE=\"true\"; zdb -R $TESTPOOL $dva:db80\
+output=$(export ZDB_NO_ZLE=\"true\"; run_zdb -e "-R" -p "$TESTPOOL $dva:db80"\
     2> /dev/null)
 output=${output##*>}
 output=${output##+([[:space:]])}
 if [ "$output" != "$blk_out1" ]; then
 	log_fail "zdb -R :db80 (block 1 display/decompress) failed"
 fi
 
-output=$(export ZDB_NO_ZLE=\"true\"; zdb -R $TESTPOOL $dva:b80d\
+output=$(export ZDB_NO_ZLE=\"true\"; run_zdb -e "-R" -p "$TESTPOOL $dva:b80d"\
     2> /dev/null)
 output=${output##*>}
 output=${output##+([[:space:]])}
@@ -113,7 +113,7 @@ fi
 vdev=$(echo "$dva" | cut -d: -f1)
 offset=$(echo "$dva" | cut -d: -f2)
 output=$(export ZDB_NO_ZLE=\"true\";\
-    zdb -R $TESTPOOL $vdev:$offset:$l1_read_size:id 2> /dev/null)
+    run_zdb -e "-R" -p "$TESTPOOL $vdev:$offset:$l1_read_size:id" 2> /dev/null)
 block_cnt=$(echo "$output" | grep -c 'L0')
 if [ $block_cnt -ne $write_count ]; then
 	log_fail "zdb -R :id (indirect block display) failed"
@@ -124,7 +124,7 @@ if ! use_object_store; then
 	vdev="$vdev.0"
 	log_note "Reading from DVA $vdev:$offset:$l1_read_size"
 	output=$(export ZDB_NO_ZLE=\"true\";\
-	    zdb -R $TESTPOOL $vdev:$offset:$l1_read_size:id 2> /dev/null)
+	    run_zdb -e "-R" -p "$TESTPOOL $vdev:$offset:$l1_read_size:id" 2> /dev/null)
 	block_cnt=$(echo "$output" | grep -c 'L0')
 	if [ $block_cnt -ne $write_count ]; then
 		log_fail "zdb -R 0.0:offset:length:id (indirect block display) failed"

tests/zfs-tests/tests/functional/cli_root/zdb/zdb_object_range_neg.ksh

@@ -47,8 +47,8 @@ set -A bad_flags a b c   e   g h i j k l   n o p q r s t u v w x y   \
 
 typeset -i i=0
 while [[ $i -lt ${#bad_flags[*]} ]]; do
-	log_mustnot zdb -dd $TESTPOOL 0:1:${bad_flags[i]}
-	log_mustnot zdb -dd $TESTPOOL 0:1:A-${bad_flags[i]}
+	log_mustnot run_zdb -e "-dd" -p "$TESTPOOL 0:1:${bad_flags[i]}"
+	log_mustnot run_zdb -e "-dd" -p "$TESTPOOL 0:1:A-${bad_flags[i]}"
 	((i = i + 1))
 done
 
@@ -58,14 +58,14 @@ set -A bad_ranges ":" "::" ":::" ":0" "0:" "0:1:" "0:1::" "0::f" "0a:1" \
 
 i=0
 while [[ $i -lt ${#bad_ranges[*]} ]]; do
-	log_mustnot zdb -dd $TESTPOOL ${bad_ranges[i]}
+	log_mustnot run_zdb -e "-dd" -p "$TESTPOOL ${bad_ranges[i]}"
 	((i = i + 1))
 done
 
 # Specifying a non-existent object identifier returns an error
-obj_id_highest=$(zdb -P -dd $TESTPOOL/$TESTFS 2>/dev/null |
+obj_id_highest=$(run_zdb -e "-P -dd" -p "$TESTPOOL/$TESTFS" 2>/dev/null |
     grep -E "^ +-?([0-9]+ +){7}" | sort -n | awk 'END {print $1}')
 obj_id_invalid=$(( $obj_id_highest + 1 ))
-log_mustnot zdb -dd $TESTPOOL/$TESTFS $obj_id_invalid
+log_mustnot run_zdb -e "-dd" -p "$TESTPOOL/$TESTFS $obj_id_invalid"
 
 log_pass "Badly formed zdb object range parameters fail as expected."

tests/zfs-tests/tests/functional/cli_root/zdb/zdb_objset_id.ksh

@@ -81,7 +81,7 @@ do
 
 	obj=$(printf "0x%X" $obj)
 	log_note "zdb -NNNNNN $TESTPOOL/$id $obj"
-	output=$(zdb -NNNNNN $TESTPOOL/$id $obj)
+	output=$(run_zdb -e "-NNNNNN" -p "$TESTPOOL/$id $obj")
 	echo $output | grep -q "$TESTPOOL/$TESTFS" ||
 		log_fail "zdb -NNNNNN $TESTPOOL/$id $obj failed ($TESTPOOL/$TESTFS not in zdb output)"
 	echo $output | grep -q "file1" ||
@@ -98,8 +98,8 @@ fi
 
 log_must zfs create $hex_ds
 log_must zfs create $num_ds
-log_must eval "zdb -d $hex_ds | grep -q \"$hex_ds\""
-log_must eval "zdb -d $num_ds | grep -q \"$num_ds\""
+log_must eval "run_zdb -e \"-d\" -p \"$hex_ds\" | grep -q \"$hex_ds\""
+log_must eval "run_zdb -e \"-d\" -p \"$num_ds\" | grep -q \"$num_ds\""
 
 # force numeric interpretation, expect fail
 log_mustnot zdb -N $hex_ds

tests/zfs-tests/tests/functional/pool_checkpoint/checkpoint_big_rewind.ksh

@@ -53,6 +53,6 @@ log_must zpool export $NESTEDPOOL
 log_must zpool import -d $FILEDISKDIR --rewind-to-checkpoint $NESTEDPOOL
 
 log_must zpool export $NESTEDPOOL
-log_must zdb -e -p $FILEDISKDIR $NESTEDPOOL
+log_must run_zdb -e "-e -p" -p "$FILEDISKDIR $NESTEDPOOL"
 
 log_pass "Rewind to checkpoint on a stressed pool."

tests/zfs-tests/tests/functional/pool_checkpoint/checkpoint_capacity.ksh

@@ -81,13 +81,13 @@ log_mustnot dd if=/dev/urandom of=$NESTEDFS0FILE bs=1M count=300
 log_must zpool list $NESTEDPOOL
 
 log_must zpool export $NESTEDPOOL
-log_must zdb -e -p $FILEDISKDIR -kc $NESTEDPOOL
+log_must run_zdb -e "-e -p" -p "$FILEDISKDIR -kc $NESTEDPOOL"
 
 log_must zpool import -d $FILEDISKDIR --rewind-to-checkpoint $NESTEDPOOL
 
 log_must [ "$(head -c 100 $NESTEDFS0FILE)" = "$FILE0INTRO" ]
 
 log_must zpool export $NESTEDPOOL
-log_must zdb -e -p $FILEDISKDIR $NESTEDPOOL
+log_must run_zdb -e "-e -p" -p "$FILEDISKDIR $NESTEDPOOL"
 
 log_pass "Do not reuse checkpointed space at low capacity."

tests/zfs-tests/tests/functional/pool_checkpoint/checkpoint_discard_busy.ksh

@@ -80,7 +80,7 @@ log_must zpool export $NESTEDPOOL
 #
 # Verify on-disk state while pool is exported
 #
-log_must zdb -e -p $FILEDISKDIR $NESTEDPOOL
+log_must run_zdb -e "-e -p" -p "$FILEDISKDIR $NESTEDPOOL"
 
 #
 # Attempt to rewind on a pool that is discarding
@@ -105,7 +105,7 @@ set_tunable64 SPA_DISCARD_MEMORY_LIMIT 16777216
 nested_wait_discard_finish
 
 log_must zpool export $NESTEDPOOL
-log_must zdb -e -p $FILEDISKDIR $NESTEDPOOL
+log_must run_zdb "-e -p" -p "$FILEDISKDIR $NESTEDPOOL"
 
 log_pass "Can export/import but not rewind/checkpoint/discard or " \
     "change pool's config while discarding."

tests/zfs-tests/tests/functional/pool_checkpoint/checkpoint_zdb.ksh

@@ -58,10 +58,10 @@ log_must zpool checkpoint $TESTPOOL
 
 test_change_state_after_checkpoint
 
-log_must eval "zdb $TESTPOOL | grep -q \"Checkpointed uberblock found\""
-log_mustnot eval "zdb -k $TESTPOOL | grep -q \"Checkpointed uberblock found\""
-log_mustnot eval "zdb $TESTPOOL | grep \"Dataset $FS1\""
-log_must eval "zdb -k $TESTPOOL | grep \"Dataset $CHECKPOINTED_FS1\""
+log_must eval "run_zdb -p $TESTPOOL | grep -q \"Checkpointed uberblock found\""
+log_mustnot eval "run_zdb -e \"-k\" -p $TESTPOOL | grep -q \"Checkpointed uberblock found\""
+log_mustnot eval "run_zdb -p $TESTPOOL | grep \"Dataset $FS1\""
+log_must eval "run_zdb -e \"-k\" -p $TESTPOOL | grep \"Dataset $CHECKPOINTED_FS1\""
 
 log_must zpool export $TESTPOOL
 
@@ -74,7 +74,7 @@ log_must import_pool -p $TESTPOOL
 
 log_must zpool checkpoint -d $TESTPOOL
 
-log_mustnot eval "zdb $TESTPOOL | grep \"Checkpointed uberblock found\""
-log_mustnot eval "zdb -k $TESTPOOL"
+log_mustnot eval "run_zdb -p $TESTPOOL | grep \"Checkpointed uberblock found\""
+log_mustnot eval "run_zdb -e \"-k\" -p $TESTPOOL"
 
 log_pass "zdb can analyze checkpointed pools."

tests/zfs-tests/tests/functional/zcache_slog_sharing/cleanup.ksh

@@ -39,7 +39,11 @@ function destroy_all_partitions
     done
 }
 
-invalidate_zcache
+#
+# Remove all devices from the zoa and destroy the
+# partitions
+#
+zcache_remove_devices $CACHE_DEVICES
 destroy_all_partitions
 
 log_pass

tests/zfs-tests/tests/functional/zcache_slog_sharing/setup.ksh

@@ -60,7 +60,7 @@ for device in $AVAILABLE_DEVICES; do
 done
 
 
-log_must configure_zettacache $CACHE_DEVICES
+log_must zcache_add_devices $CACHE_DEVICES
 verify_zcache_devices_were_added $CACHE_DEVICES
 
 log_note "Available devices [$AVAILABLE_DEVICES]"

tests/zfs-tests/tests/functional/zcache_slog_sharing/zcache_slog_sharing_import_pool_slog_missing_neg.ksh

@@ -60,7 +60,7 @@ fi
 function cleanup
 {
 	poolexists $TESTPOOL && destroy_pool $TESTPOOL
-	invalidate_zcache
+	zcache_add_devices $CACHE_DEVICES
 }
 
 log_assert "Verify that the shared device(s) are partitioned into two parts." \
@@ -84,8 +84,8 @@ verify_slog_devices_are_online $TESTPOOL $SLOG_DEVICES
 
 log_must zpool export $TESTPOOL
 
-invalidate_zcache
-
+# Clear the labels so that the partition can be destroyed
+zcache_remove_devices $CACHE_DEVICES
 for dev in ${AVAILABLE_DEVICES}; do
 	# Partition 1 is the slog part
 	destroy_partition $dev 1

tests/zfs-tests/tests/functional/zcache_slog_sharing/zcache_slog_sharing_multiple_pools_pos.ksh

@@ -59,7 +59,6 @@ function cleanup
 {
 	poolexists $TESTPOOL && destroy_pool $TESTPOOL
 	poolexists $TESTPOOL1 && destroy_pool $TESTPOOL1
-	invalidate_zcache
 }
 
 log_assert "Verify that the shared device(s) are partitioned into two parts." \

tests/zfs-tests/tests/functional/zcache_slog_sharing/zcache_slog_sharing_slog_same_as_cache_neg.ksh

@@ -56,7 +56,6 @@ fi
 function cleanup
 {
 	poolexists $TESTPOOL && destroy_pool $TESTPOOL
-	invalidate_zcache
 }
 
 log_assert "Verify that the shared device(s) are partitioned into two parts." \