Skip to content

Commit

Permalink
Refactor compression algorithm selection for tests
Browse files Browse the repository at this point in the history
- Moves compression algorithms for tests to properties.shlib
- Removes all compression algorithms levels from general tests
- Replaces on with lz4 for compression tests
- Removes random algorithm selection, if not needed
- Cleans copyright header formatting

Reviewed-by: Brian Behlendorf <[email protected]>
Reviewed-by: John Kennedy <[email protected]>
Reviewed-by: Michael Niewöhner <[email protected]>
Signed-off-by: Kjeld Schouten-Lebbing <[email protected]>
Closes #9645
Kjeld Schouten authored and behlendorf committed Dec 4, 2019
1 parent 5a08977 commit 618b6ad
Showing 13 changed files with 50 additions and 91 deletions.
38 changes: 7 additions & 31 deletions tests/zfs-tests/include/libtest.shlib
Original file line number Diff line number Diff line change
@@ -20,14 +20,13 @@
#

#
# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
# Use is subject to license terms.
# Copyright (c) 2012, 2018 by Delphix. All rights reserved.
# Copyright (c) 2017 by Tim Chase. All rights reserved.
# Copyright (c) 2017 by Nexenta Systems, Inc. All rights reserved.
# Copyright (c) 2017 Lawrence Livermore National Security, LLC.
# Copyright (c) 2017 Datto Inc.
# Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
# Copyright (c) 2009, Sun Microsystems Inc. All rights reserved.
# Copyright (c) 2012, 2018, Delphix. All rights reserved.
# Copyright (c) 2017, Tim Chase. All rights reserved.
# Copyright (c) 2017, Nexenta Systems Inc. All rights reserved.
# Copyright (c) 2017, Lawrence Livermore National Security LLC.
# Copyright (c) 2017, Datto Inc. All rights reserved.
# Copyright (c) 2017, Open-E Inc. All rights reserved.
# Use is subject to license terms.
#

@@ -2530,29 +2529,6 @@ function safe_to_destroy_pool { # $1 the pool name
fi
}

#
# Get the available ZFS compression options
# $1 option type zfs_set|zfs_compress
#
function get_compress_opts
{
typeset COMPRESS_OPTS
typeset GZIP_OPTS="gzip gzip-1 gzip-2 gzip-3 gzip-4 gzip-5 \
gzip-6 gzip-7 gzip-8 gzip-9"

if [[ $1 == "zfs_compress" ]] ; then
COMPRESS_OPTS="on lzjb"
elif [[ $1 == "zfs_set" ]] ; then
COMPRESS_OPTS="on off lzjb"
fi
typeset valid_opts="$COMPRESS_OPTS"
zfs get 2>&1 | grep gzip >/dev/null 2>&1
if [[ $? -eq 0 ]]; then
valid_opts="$valid_opts $GZIP_OPTS"
fi
echo "$valid_opts"
}

#
# Verify zfs operation with -p option work as expected
# $1 operation, value could be create, clone or rename
15 changes: 2 additions & 13 deletions tests/zfs-tests/include/properties.shlib
Original file line number Diff line number Diff line change
@@ -10,11 +10,10 @@
#

#
# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
# Copyright (c) 2012, 2016, Delphix. All rights reserved.
#

typeset -a compress_prop_vals=('on' 'off' 'lzjb' 'gzip' 'gzip-1' 'gzip-2'
'gzip-3' 'gzip-4' 'gzip-5' 'gzip-6' 'gzip-7' 'gzip-8' 'gzip-9' 'zle' 'lz4')
typeset -a compress_prop_vals=('off' 'lzjb' 'lz4' 'gzip' 'zle')
typeset -a checksum_prop_vals=('on' 'off' 'fletcher2' 'fletcher4' 'sha256'
'noparity' 'sha512' 'skein' 'edonr')
typeset -a recsize_prop_vals=('512' '1024' '2048' '4096' '8192' '16384'
@@ -58,16 +57,6 @@ function get_rand_prop
echo $retstr
}

function get_rand_compress
{
get_rand_prop compress_prop_vals $1 2
}

function get_rand_compress_any
{
get_rand_prop compress_prop_vals $1 0
}

function get_rand_checksum
{
get_rand_prop checksum_prop_vals $1 2
Original file line number Diff line number Diff line change
@@ -26,6 +26,7 @@
#

. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/include/properties.shlib
. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib

#
@@ -41,7 +42,7 @@
verify_runnable "both"

set -A dataset "$TESTPOOL" "$TESTPOOL/$TESTFS" "$TESTPOOL/$TESTVOL"
set -A values $(get_compress_opts zfs_set)
set -A values "${compress_prop_vals[@]}"

log_assert "Setting a valid compression on file system and volume, " \
"It should be successful."
Original file line number Diff line number Diff line change
@@ -21,14 +21,12 @@
#

#
# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2009, Sun Microsystems Inc. All rights reserved.
# Copyright (c) 2016, 2017, Delphix. All rights reserved.
# Use is subject to license terms.
#

#
# Copyright (c) 2016, 2017 by Delphix. All rights reserved.
#

. $STF_SUITE/include/properties.shlib
. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib

@@ -106,7 +104,7 @@ for ds in $pool $fs $vol; do
done
;;
compression|compress )
for val in $(get_compress_opts zfs_set); do
for val in "${compress_prop_vals[@]}"; do
set_and_check $ds ${rw_prop[i]} $val ${chk_prop[i]}
done
;;
Original file line number Diff line number Diff line change
@@ -21,14 +21,11 @@
#

#
# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2007, Sun Microsystems Inc. All rights reserved.
# Copyright (c) 2013, 2016, Delphix. All rights reserved.
# Use is subject to license terms.
#

#
# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
#

. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/compression/compress.cfg

Original file line number Diff line number Diff line change
@@ -21,14 +21,14 @@
#

#
# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2007, Sun Microsystems Inc. All rights reserved.
# Copyright (c) 2013, 2016, Delphix. All rights reserved.
# Copyright (c) 2019, Kjeld Schouten-Lebbing. All rights reserved.
# Use is subject to license terms.
#

#
# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
#

. $STF_SUITE/include/properties.shlib
. $STF_SUITE/include/libtest.shlib

#
@@ -62,7 +62,7 @@ typeset -i offset=0

for propname in "compression" "compress"
do
for value in $(get_compress_opts zfs_compress)
for value in "${compress_prop_vals[@]:1}"
do
log_must zfs set $propname=$value $fs
if [[ $value == "gzip-6" ]]; then
Original file line number Diff line number Diff line change
@@ -21,14 +21,13 @@
#

#
# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2007, Sun Microsystems Inc. All rights reserved.
# Copyright (c) 2013, 2016, Delphix. All rights reserved.
# Copyright (c) 2019, Kjeld Schouten-Lebbing. All Rights Reserved.
# Use is subject to license terms.
#

#
# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
#

. $STF_SUITE/include/properties.shlib
. $STF_SUITE/include/libtest.shlib

#
@@ -94,7 +93,7 @@ typeset -i blknum=0

for propname in "compression" "compress"
do
for value in $(get_compress_opts zfs_compress)
for value in "${compress_prop_vals[@]:1}"
do
log_must zfs set compression=$value $fs
real_val=$(get_prop $propname $fs)
Original file line number Diff line number Diff line change
@@ -12,11 +12,12 @@
#

#
# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
# Copyright (c) 2012, 2016, Delphix. All rights reserved.
# Copyright (c) 2019, Kjeld Schouten-Lebbing. All Rights Reserved.
#

. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/include/properties.shlib
. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/nopwrite/nopwrite.shlib

#
@@ -51,8 +52,8 @@ log_must zfs set checksum=sha256 $origin
dd if=/dev/urandom of=$TESTDIR/file bs=1024k count=$MEGS conv=notrunc \
>/dev/null 2>&1 || log_fail "initial dd failed."

# Verify nop_write for 4 random compression algorithms
for i in $(get_rand_compress 4); do
# Verify nop_write for all compression algorithms except "off"
for i in "${compress_prop_vals[@]:1}"; do
zfs snapshot $origin@a || log_fail "zfs snap failed"
log_must zfs clone -o compress=$i $origin@a $origin/clone
dd if=/$TESTDIR/file of=/$TESTDIR/clone/file bs=1024k count=$MEGS \
15 changes: 5 additions & 10 deletions tests/zfs-tests/tests/functional/rsend/rsend_012_pos.ksh
Original file line number Diff line number Diff line change
@@ -21,14 +21,13 @@
#

#
# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2009, Sun Microsystems Inc. All rights reserved.
# Copyright (c) 2013, 2016, Delphix. All rights reserved.
# Use is subject to license terms.
#

#
# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
#

. $STF_SUITE/include/properties.shlib
. $STF_SUITE/tests/functional/rsend/rsend.kshlib

#
@@ -121,9 +120,7 @@ for fs in "$POOL" "$POOL/pclone" "$POOL/$FS" "$POOL/$FS/fs1" \
rand_set_prop $fs acltype "off" "noacl" "posixacl"
rand_set_prop $fs atime "on" "off"
rand_set_prop $fs checksum "on" "off" "fletcher2" "fletcher4" "sha256"
rand_set_prop $fs compression "on" "off" "lzjb" "gzip" \
"gzip-1" "gzip-2" "gzip-3" "gzip-4" "gzip-5" "gzip-6" \
"gzip-7" "gzip-8" "gzip-9"
rand_set_prop $fs compression "${compress_prop_vals[@]}"
rand_set_prop $fs copies "1" "2" "3"
rand_set_prop $fs devices "on" "off"
rand_set_prop $fs exec "on" "off"
@@ -138,9 +135,7 @@ done

for vol in "$POOL/vol" "$POOL/$FS/vol" ; do
rand_set_prop $vol checksum "on" "off" "fletcher2" "fletcher4" "sha256"
rand_set_prop $vol compression "on" "off" "lzjb" "gzip" \
"gzip-1" "gzip-2" "gzip-3" "gzip-4" "gzip-5" "gzip-6" \
"gzip-7" "gzip-8" "gzip-9"
rand_set_prop $vol compression "${compress_prop_vals[@]}"
rand_set_prop $vol readonly "on" "off"
rand_set_prop $vol copies "1" "2" "3"
rand_set_prop $vol user:prop "aaa" "bbb" "23421" "()-+?"
Original file line number Diff line number Diff line change
@@ -12,10 +12,11 @@
#

#
# Copyright (c) 2015 by Delphix. All rights reserved.
# Copyright (c) 2015, Delphix. All rights reserved.
#

. $STF_SUITE/tests/functional/rsend/rsend.kshlib
. $STF_SUITE/include/properties.shlib

#
# Description:
@@ -34,7 +35,6 @@ verify_runnable "both"

log_assert "Verify compressed streams are rejected if incompatible."

typeset compress_types="off gzip lz4"
typeset send_ds=$POOL2/testds
typeset recv_ds=$POOL3/testds

@@ -49,7 +49,7 @@ log_onexit cleanup
datasetexists $POOL3 && log_must zpool destroy $POOL3
log_must zpool create -d $POOL3 $DISK3

for compress in $compress_types; do
for compress in "${compress_prop_vals[@]}"; do
datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds
datasetexists $recv_ds && log_must_busy zfs destroy -r $recv_ds

Original file line number Diff line number Diff line change
@@ -12,10 +12,11 @@
#

#
# Copyright (c) 2015 by Delphix. All rights reserved.
# Copyright (c) 2015, Delphix. All rights reserved.
#

. $STF_SUITE/tests/functional/rsend/rsend.kshlib
. $STF_SUITE/include/properties.shlib

#
# Description:
@@ -28,7 +29,6 @@
#

verify_runnable "both"
typeset compress_types="off gzip lz4"
typeset send_ds="$POOL2/testfs"
typeset send_vol="$POOL2/vol"
typeset send_voldev="$ZVOL_DEVDIR/$POOL2/vol"
@@ -55,7 +55,7 @@ log_onexit cleanup_pool $POOL2

write_compressible $BACKDIR ${megs}m

for compress in $compress_types; do
for compress in "${compress_prop_vals[@]}"; do
datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds
datasetexists $send_vol && log_must_busy zfs destroy -r $send_vol
log_must zfs create -o compress=$compress $send_ds
Original file line number Diff line number Diff line change
@@ -12,7 +12,8 @@
#

#
# Copyright (c) 2015 by Delphix. All rights reserved.
# Copyright (c) 2015, Delphix. All rights reserved.
# Copyright (c) 2019, Kjeld Schouten-Lebbing. All rights reserved.
#

. $STF_SUITE/tests/functional/rsend/rsend.kshlib
@@ -37,7 +38,7 @@ log_onexit cleanup_pool $POOL2
typeset sendfs=$POOL2/$FS
typeset megs=128

for prop in $(get_rand_compress_any 6); do
for prop in "${compress_prop_vals[@]}"; do
for compressible in 'yes' 'no'; do
log_must zfs create -o compress=$prop $sendfs

Original file line number Diff line number Diff line change
@@ -12,9 +12,11 @@
#

#
# Copyright (c) 2019 by Lawrence Livermore National Security, LLC.
# Copyright (c) 2019, Lawrence Livermore National Security LLC.
# Use is subject to license terms.
#

. $STF_SUITE/include/properties.shlib
. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/rsend/rsend.kshlib

@@ -78,7 +80,7 @@ for i in {1..$passes}; do
# Randomly modify several dataset properties in order to generate
# more interesting incremental send streams.
rand_set_prop $POOL/fs checksum "off" "fletcher4" "sha256"
rand_set_prop $POOL/fs compression "off" "lzjb" "gzip" "lz4"
rand_set_prop $POOL/fs compression "${compress_prop_vals[@]}"
rand_set_prop $POOL/fs recordsize "32K" "128K"
rand_set_prop $POOL/fs dnodesize "legacy" "auto" "4k"
rand_set_prop $POOL/fs xattr "on" "sa"

0 comments on commit 618b6ad

Please sign in to comment.