Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test Compression Algorithm Selection Refactor #9645

Merged
merged 1 commit into from
Dec 4, 2019
Merged

Test Compression Algorithm Selection Refactor #9645

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 7 additions & 31 deletions tests/zfs-tests/include/libtest.shlib
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,13 @@
#

#
# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
# Use is subject to license terms.
# Copyright (c) 2012, 2018 by Delphix. All rights reserved.
# Copyright (c) 2017 by Tim Chase. All rights reserved.
# Copyright (c) 2017 by Nexenta Systems, Inc. All rights reserved.
# Copyright (c) 2017 Lawrence Livermore National Security, LLC.
# Copyright (c) 2017 Datto Inc.
# Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
# Copyright (c) 2009, Sun Microsystems Inc. All rights reserved.
# Copyright (c) 2012, 2018, Delphix. All rights reserved.
# Copyright (c) 2017, Tim Chase. All rights reserved.
# Copyright (c) 2017, Nexenta Systems Inc. All rights reserved.
# Copyright (c) 2017, Lawrence Livermore National Security LLC.
# Copyright (c) 2017, Datto Inc. All rights reserved.
# Copyright (c) 2017, Open-E Inc. All rights reserved.
# Use is subject to license terms.
#

Expand Down Expand Up @@ -2530,29 +2529,6 @@ function safe_to_destroy_pool { # $1 the pool name
fi
}

#
# Get the available ZFS compression options
# $1 option type zfs_set|zfs_compress
#
function get_compress_opts
{
typeset COMPRESS_OPTS
typeset GZIP_OPTS="gzip gzip-1 gzip-2 gzip-3 gzip-4 gzip-5 \
gzip-6 gzip-7 gzip-8 gzip-9"

if [[ $1 == "zfs_compress" ]] ; then
COMPRESS_OPTS="on lzjb"
elif [[ $1 == "zfs_set" ]] ; then
COMPRESS_OPTS="on off lzjb"
fi
typeset valid_opts="$COMPRESS_OPTS"
zfs get 2>&1 | grep gzip >/dev/null 2>&1
if [[ $? -eq 0 ]]; then
valid_opts="$valid_opts $GZIP_OPTS"
fi
echo "$valid_opts"
}

#
# Verify zfs operation with -p option work as expected
# $1 operation, value could be create, clone or rename
Expand Down
15 changes: 2 additions & 13 deletions tests/zfs-tests/include/properties.shlib
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,10 @@
#

#
# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
# Copyright (c) 2012, 2016, Delphix. All rights reserved.
#

typeset -a compress_prop_vals=('on' 'off' 'lzjb' 'gzip' 'gzip-1' 'gzip-2'
'gzip-3' 'gzip-4' 'gzip-5' 'gzip-6' 'gzip-7' 'gzip-8' 'gzip-9' 'zle' 'lz4')
typeset -a compress_prop_vals=('off' 'lzjb' 'lz4' 'gzip' 'zle')
jwk404 marked this conversation as resolved.
Show resolved Hide resolved
typeset -a checksum_prop_vals=('on' 'off' 'fletcher2' 'fletcher4' 'sha256'
'noparity' 'sha512' 'skein' 'edonr')
typeset -a recsize_prop_vals=('512' '1024' '2048' '4096' '8192' '16384'
Expand Down Expand Up @@ -58,16 +57,6 @@ function get_rand_prop
echo $retstr
}

function get_rand_compress
{
get_rand_prop compress_prop_vals $1 2
}

function get_rand_compress_any
{
get_rand_prop compress_prop_vals $1 0
}

function get_rand_checksum
{
get_rand_prop checksum_prop_vals $1 2
Expand Down
3 changes: 2 additions & 1 deletion tests/zfs-tests/tests/functional/cli_root/zfs_set/compression_001_pos.ksh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#

. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/include/properties.shlib
. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib

#
Expand All @@ -41,7 +42,7 @@
verify_runnable "both"

set -A dataset "$TESTPOOL" "$TESTPOOL/$TESTFS" "$TESTPOOL/$TESTVOL"
set -A values $(get_compress_opts zfs_set)
set -A values "${compress_prop_vals[@]}"

log_assert "Setting a valid compression on file system and volume, " \
"It should be successful."
Expand Down
10 changes: 4 additions & 6 deletions tests/zfs-tests/tests/functional/cli_root/zfs_set/property_alias_001_pos.ksh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,12 @@
#

#
# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2009, Sun Microsystems Inc. All rights reserved.
# Copyright (c) 2016, 2017, Delphix. All rights reserved.
# Use is subject to license terms.
#

#
# Copyright (c) 2016, 2017 by Delphix. All rights reserved.
#

. $STF_SUITE/include/properties.shlib
. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/cli_root/zfs_set/zfs_set_common.kshlib

Expand Down Expand Up @@ -106,7 +104,7 @@ for ds in $pool $fs $vol; do
done
;;
compression|compress )
for val in $(get_compress_opts zfs_set); do
for val in "${compress_prop_vals[@]}"; do
set_and_check $ds ${rw_prop[i]} $val ${chk_prop[i]}
done
;;
Expand Down
7 changes: 2 additions & 5 deletions tests/zfs-tests/tests/functional/compression/compress_001_pos.ksh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,11 @@
#

#
# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2007, Sun Microsystems Inc. All rights reserved.
# Copyright (c) 2013, 2016, Delphix. All rights reserved.
# Use is subject to license terms.
#

#
# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
#

. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/compression/compress.cfg

Expand Down
10 changes: 5 additions & 5 deletions tests/zfs-tests/tests/functional/compression/compress_003_pos.ksh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,14 @@
#

#
# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2007, Sun Microsystems Inc. All rights reserved.
# Copyright (c) 2013, 2016, Delphix. All rights reserved.
# Copyright (c) 2019, Kjeld Schouten-Lebbing. All rights reserved.
# Use is subject to license terms.
#

#
# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
#

. $STF_SUITE/include/properties.shlib
. $STF_SUITE/include/libtest.shlib

#
Expand Down Expand Up @@ -62,7 +62,7 @@ typeset -i offset=0

for propname in "compression" "compress"
do
for value in $(get_compress_opts zfs_compress)
for value in "${compress_prop_vals[@]:1}"
PrivatePuffin marked this conversation as resolved.
Show resolved Hide resolved
do
log_must zfs set $propname=$value $fs
if [[ $value == "gzip-6" ]]; then
Expand Down
11 changes: 5 additions & 6 deletions tests/zfs-tests/tests/functional/compression/compress_004_pos.ksh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,13 @@
#

#
# Copyright 2007 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2007, Sun Microsystems Inc. All rights reserved.
# Copyright (c) 2013, 2016, Delphix. All rights reserved.
# Copyright (c) 2019, Kjeld Schouten-Lebbing. All Rights Reserved.
# Use is subject to license terms.
#

#
# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
#

. $STF_SUITE/include/properties.shlib
. $STF_SUITE/include/libtest.shlib

#
Expand Down Expand Up @@ -94,7 +93,7 @@ typeset -i blknum=0

for propname in "compression" "compress"
do
for value in $(get_compress_opts zfs_compress)
for value in "${compress_prop_vals[@]:1}"
PrivatePuffin marked this conversation as resolved.
Show resolved Hide resolved
do
log_must zfs set compression=$value $fs
real_val=$(get_prop $propname $fs)
Expand Down
9 changes: 5 additions & 4 deletions tests/zfs-tests/tests/functional/nopwrite/nopwrite_varying_compression.ksh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,12 @@
#

#
# Copyright (c) 2012, 2016 by Delphix. All rights reserved.
# Copyright (c) 2012, 2016, Delphix. All rights reserved.
# Copyright (c) 2019, Kjeld Schouten-Lebbing. All Rights Reserved.
#

. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/include/properties.shlib
. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/nopwrite/nopwrite.shlib

#
Expand Down Expand Up @@ -51,8 +52,8 @@ log_must zfs set checksum=sha256 $origin
dd if=/dev/urandom of=$TESTDIR/file bs=1024k count=$MEGS conv=notrunc \
>/dev/null 2>&1 || log_fail "initial dd failed."

# Verify nop_write for 4 random compression algorithms
for i in $(get_rand_compress 4); do
# Verify nop_write for all compression algorithms except "off"
for i in "${compress_prop_vals[@]:1}"; do
zfs snapshot $origin@a || log_fail "zfs snap failed"
log_must zfs clone -o compress=$i $origin@a $origin/clone
dd if=/$TESTDIR/file of=/$TESTDIR/clone/file bs=1024k count=$MEGS \
Expand Down
15 changes: 5 additions & 10 deletions tests/zfs-tests/tests/functional/rsend/rsend_012_pos.ksh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,13 @@
#

#
# Copyright 2009 Sun Microsystems, Inc. All rights reserved.
# Copyright (c) 2009, Sun Microsystems Inc. All rights reserved.
# Copyright (c) 2013, 2016, Delphix. All rights reserved.
# Use is subject to license terms.
#

#
# Copyright (c) 2013, 2016 by Delphix. All rights reserved.
#

. $STF_SUITE/include/properties.shlib
. $STF_SUITE/tests/functional/rsend/rsend.kshlib

#
Expand Down Expand Up @@ -121,9 +120,7 @@ for fs in "$POOL" "$POOL/pclone" "$POOL/$FS" "$POOL/$FS/fs1" \
rand_set_prop $fs acltype "off" "noacl" "posixacl"
rand_set_prop $fs atime "on" "off"
rand_set_prop $fs checksum "on" "off" "fletcher2" "fletcher4" "sha256"
rand_set_prop $fs compression "on" "off" "lzjb" "gzip" \
"gzip-1" "gzip-2" "gzip-3" "gzip-4" "gzip-5" "gzip-6" \
"gzip-7" "gzip-8" "gzip-9"
rand_set_prop $fs compression "${compress_prop_vals[@]}"
rand_set_prop $fs copies "1" "2" "3"
rand_set_prop $fs devices "on" "off"
rand_set_prop $fs exec "on" "off"
Expand All @@ -138,9 +135,7 @@ done

for vol in "$POOL/vol" "$POOL/$FS/vol" ; do
rand_set_prop $vol checksum "on" "off" "fletcher2" "fletcher4" "sha256"
rand_set_prop $vol compression "on" "off" "lzjb" "gzip" \
"gzip-1" "gzip-2" "gzip-3" "gzip-4" "gzip-5" "gzip-6" \
"gzip-7" "gzip-8" "gzip-9"
rand_set_prop $vol compression "${compress_prop_vals[@]}"
rand_set_prop $vol readonly "on" "off"
rand_set_prop $vol copies "1" "2" "3"
rand_set_prop $vol user:prop "aaa" "bbb" "23421" "()-+?"
Expand Down
6 changes: 3 additions & 3 deletions tests/zfs-tests/tests/functional/rsend/send-c_recv_lz4_disabled.ksh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,11 @@
#

#
# Copyright (c) 2015 by Delphix. All rights reserved.
# Copyright (c) 2015, Delphix. All rights reserved.
#

. $STF_SUITE/tests/functional/rsend/rsend.kshlib
. $STF_SUITE/include/properties.shlib

#
# Description:
Expand All @@ -34,7 +35,6 @@ verify_runnable "both"

log_assert "Verify compressed streams are rejected if incompatible."

typeset compress_types="off gzip lz4"
typeset send_ds=$POOL2/testds
typeset recv_ds=$POOL3/testds

Expand All @@ -49,7 +49,7 @@ log_onexit cleanup
datasetexists $POOL3 && log_must zpool destroy $POOL3
log_must zpool create -d $POOL3 $DISK3

for compress in $compress_types; do
for compress in "${compress_prop_vals[@]}"; do
datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds
datasetexists $recv_ds && log_must_busy zfs destroy -r $recv_ds

Expand Down
6 changes: 3 additions & 3 deletions tests/zfs-tests/tests/functional/rsend/send-c_stream_size_estimate.ksh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,11 @@
#

#
# Copyright (c) 2015 by Delphix. All rights reserved.
# Copyright (c) 2015, Delphix. All rights reserved.
#

. $STF_SUITE/tests/functional/rsend/rsend.kshlib
. $STF_SUITE/include/properties.shlib

#
# Description:
Expand All @@ -28,7 +29,6 @@
#

verify_runnable "both"
typeset compress_types="off gzip lz4"
typeset send_ds="$POOL2/testfs"
typeset send_vol="$POOL2/vol"
typeset send_voldev="$ZVOL_DEVDIR/$POOL2/vol"
Expand All @@ -55,7 +55,7 @@ log_onexit cleanup_pool $POOL2

write_compressible $BACKDIR ${megs}m

for compress in $compress_types; do
for compress in "${compress_prop_vals[@]}"; do
datasetexists $send_ds && log_must_busy zfs destroy -r $send_ds
datasetexists $send_vol && log_must_busy zfs destroy -r $send_vol
log_must zfs create -o compress=$compress $send_ds
Expand Down
5 changes: 3 additions & 2 deletions tests/zfs-tests/tests/functional/rsend/send-c_verify_ratio.ksh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@
#

#
# Copyright (c) 2015 by Delphix. All rights reserved.
# Copyright (c) 2015, Delphix. All rights reserved.
# Copyright (c) 2019, Kjeld Schouten-Lebbing. All rights reserved.
#

. $STF_SUITE/tests/functional/rsend/rsend.kshlib
Expand All @@ -37,7 +38,7 @@ log_onexit cleanup_pool $POOL2
typeset sendfs=$POOL2/$FS
typeset megs=128

for prop in $(get_rand_compress_any 6); do
for prop in "${compress_prop_vals[@]}"; do
for compressible in 'yes' 'no'; do
log_must zfs create -o compress=$prop $sendfs

Expand Down
6 changes: 4 additions & 2 deletions tests/zfs-tests/tests/functional/rsend/send_realloc_encrypted_files.ksh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@
#

#
# Copyright (c) 2019 by Lawrence Livermore National Security, LLC.
# Copyright (c) 2019, Lawrence Livermore National Security LLC.
# Use is subject to license terms.
#

. $STF_SUITE/include/properties.shlib
. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/rsend/rsend.kshlib

Expand Down Expand Up @@ -78,7 +80,7 @@ for i in {1..$passes}; do
# Randomly modify several dataset properties in order to generate
# more interesting incremental send streams.
rand_set_prop $POOL/fs checksum "off" "fletcher4" "sha256"
rand_set_prop $POOL/fs compression "off" "lzjb" "gzip" "lz4"
rand_set_prop $POOL/fs compression "${compress_prop_vals[@]}"
rand_set_prop $POOL/fs recordsize "32K" "128K"
rand_set_prop $POOL/fs dnodesize "legacy" "auto" "4k"
rand_set_prop $POOL/fs xattr "on" "sa"
Expand Down