ZFS version: ============ [root@localhost zfs]# git log --pretty=oneline 86c1edc69decd9256b963c90bdd1527f9ea22e37 (HEAD -> zpool_clear) zed: Take no action on scrub/resilver checksum errors ======================== DRAID2 ======================== + truncate -s 1G file1 file2 file3 file4 file5 file6 file7 file8 file9 file10 file11 file12 file13 file14 file15 file16 file17 file18 file19 file20 ++ pwd /root/test/test/files + zpool create -f -o cachefile=none -o failmode=panic -o multihost=on -O canmount=off pool-oss0 draid2:16d:2s /root/test/test/files/file1 /root/test/test/files/file2 /root/test/test/files/file3 /root/test/test/files/file4 /root/test/test/files/file5 /root/test/test/files/file6 /root/test/test/files/file7 /root/test/test/files/file8 /root/test/test/files/file9 /root/test/test/files/file10 /root/test/test/files/file11 /root/test/test/files/file12 /root/test/test/files/file13 /root/test/test/files/file14 /root/test/test/files/file15 /root/test/test/files/file16 /root/test/test/files/file17 /root/test/test/files/file18 /root/test/test/files/file19 /root/test/test/files/file20 + zfs create -o mountpoint=/mnt/ost0 pool-oss0/ost0 + zfs set recordsize=4M pool-oss0 [root@localhost files]# [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# zfs list NAME USED AVAIL REFER MOUNTPOINT pool-oss0 1.20M 14.6G 192K /pool-oss0 pool-oss0/ost0 192K 14.6G 192K /mnt/ost0 [root@localhost files]# [root@localhost files]# df -h Filesystem Size Used Avail Use% Mounted on devtmpfs 3.8G 0 3.8G 0% /dev tmpfs 3.8G 0 3.8G 0% /dev/shm tmpfs 3.8G 13M 3.8G 1% /run tmpfs 3.8G 0 3.8G 0% /sys/fs/cgroup /dev/sda2 43G 15G 27G 36% / shared_folder 101G 66G 35G 66% /media/sf_shared_folder tmpfs 777M 0 777M 0% /run/user/0 pool-oss0/ost0 15G 4.0M 15G 1% /mnt/ost0 [root@localhost files]# [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=write --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) Jobs: 2 (f=1): [_(2),f(1),_(1),W(1),_(1)][98.2%][w=235MiB/s][w=235 IOPS][eta 00m:01s] test1: (groupid=0, jobs=6): err= 0: pid=5131: Thu Jun 2 21:15:26 2022 write: IOPS=218, BW=218MiB/s (229MB/s)(12.0GiB/56343msec); 0 zone resets slat (usec): min=136, max=1942.1k, avg=25768.20, stdev=135317.14 clat (usec): min=2, max=2871.7k, avg=77712.40, stdev=236414.47 lat (usec): min=165, max=2877.8k, avg=103498.37, stdev=271909.29 clat percentiles (usec): | 1.00th=[ 1369], 5.00th=[ 2442], 10.00th=[ 3130], | 20.00th=[ 4293], 30.00th=[ 5276], 40.00th=[ 6194], | 50.00th=[ 7177], 60.00th=[ 8225], 70.00th=[ 9634], | 80.00th=[ 13173], 90.00th=[ 96994], 95.00th=[ 650118], | 99.00th=[1002439], 99.50th=[1417675], 99.90th=[1937769], | 99.95th=[2197816], 99.99th=[2432697] bw ( KiB/s): min=14084, max=1224861, per=100.00%, avg=343028.77, stdev=42572.29, samples=423 iops : min= 8, max= 1193, avg=331.54, stdev=41.61, samples=423 lat (usec) : 4=0.01%, 10=0.02%, 20=0.02%, 250=0.01%, 500=0.05% lat (usec) : 750=0.11%, 1000=0.37% lat (msec) : 2=1.79%, 4=14.82%, 10=55.01%, 20=13.02%, 50=3.87% lat (msec) : 100=0.95%, 250=0.75%, 500=0.74%, 750=5.43%, 1000=2.03% lat (msec) : 2000=0.93%, >=2000=0.07% cpu : usr=0.89%, sys=6.90%, ctx=2944, majf=0, minf=82 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=0,12288,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): WRITE: bw=218MiB/s (229MB/s), 218MiB/s-218MiB/s (229MB/s-229MB/s), io=12.0GiB (12.9GB), run=56343-56343msec [root@localhost files]# [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# du -sh /mnt/ost0/* 2.0G /mnt/ost0/test1.0.0 2.0G /mnt/ost0/test1.1.0 2.0G /mnt/ost0/test1.2.0 2.0G /mnt/ost0/test1.3.0 2.0G /mnt/ost0/test1.4.0 2.0G /mnt/ost0/test1.5.0 [root@localhost files]# md5sum /mnt/ost0/* > md5sum [root@localhost files]# cat md5sum 5b59a53eb66d2d80d758c40ab13a7162 /mnt/ost0/test1.0.0 14274db8ac5a12fb73a29919edbdf739 /mnt/ost0/test1.1.0 77e6532c0d4dabe5d960235f1e9baf8c /mnt/ost0/test1.2.0 923ef8be3e2f6563deb5e41e5e34fcab /mnt/ost0/test1.3.0 a9c40611d3d84f58ea74c867ad086da6 /mnt/ost0/test1.4.0 852e8df7c63bd5dff1f06f1abdffd2fe /mnt/ost0/test1.5.0 [root@localhost files]# [root@localhost files]# [root@localhost files]# dd if=/dev/zero of=file10 bs=1M count=1024 oflag=direct 1024+0 records in 1024+0 records out 1073741824 bytes (1.1 GB, 1.0 GiB) copied, 0.56943 s, 1.9 GB/s [root@localhost files]# [root@localhost files]# [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 1 (f=1): [_(4),R(1),_(1)][100.0%][r=178MiB/s][r=178 IOPS][eta 00m:00s] test1: (groupid=0, jobs=6): err= 0: pid=23925: Thu Jun 2 21:26:13 2022 read: IOPS=275, BW=276MiB/s (289MB/s)(12.0GiB/44569msec) slat (usec): min=107, max=664573, avg=20658.25, stdev=54235.08 clat (usec): min=3, max=668963, avg=61927.17, stdev=80563.00 lat (usec): min=205, max=670929, avg=82591.93, stdev=84281.02 clat percentiles (usec): | 1.00th=[ 766], 5.00th=[ 1254], 10.00th=[ 1745], 20.00th=[ 3195], | 30.00th=[ 14353], 40.00th=[ 24249], 50.00th=[ 33817], 60.00th=[ 45876], | 70.00th=[ 62129], 80.00th=[ 90702], 90.00th=[175113], 95.00th=[248513], | 99.00th=[354419], 99.50th=[408945], 99.90th=[501220], 99.95th=[566232], | 99.99th=[666895] bw ( KiB/s): min=48172, max=988622, per=100.00%, avg=295135.34, stdev=40093.64, samples=501 iops : min= 42, max= 962, avg=284.34, stdev=39.24, samples=501 lat (usec) : 4=0.01%, 10=0.04%, 250=0.02%, 500=0.09%, 750=0.78% lat (usec) : 1000=1.82% lat (msec) : 2=9.90%, 4=8.95%, 10=4.86%, 20=9.34%, 50=26.82% lat (msec) : 100=18.92%, 250=13.47%, 500=4.85%, 750=0.12% cpu : usr=0.13%, sys=3.48%, ctx=5698, majf=1, minf=3191 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=276MiB/s (289MB/s), 276MiB/s-276MiB/s (289MB/s-289MB/s), io=12.0GiB (12.9GB), run=44569-44569msec [root@localhost files]# [root@localhost files]# md5sum -c md5sum /mnt/ost0/test1.0.0: OK /mnt/ost0/test1.1.0: OK /mnt/ost0/test1.2.0: OK /mnt/ost0/test1.3.0: OK /mnt/ost0/test1.4.0: OK /mnt/ost0/test1.5.0: OK [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an unrecoverable error. An attempt was made to correct the error. Applications are unaffected. action: Determine if the device needs to be replaced, and clear the errors using 'zpool clear' or replace the device with 'zpool replace'. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-9P scan: scrub repaired 0B in 00:00:30 with 0 errors on Thu Jun 2 21:26:26 2022 scan: resilvered (draid2:16d:20c:2s-0) 688M in 00:00:25 with 0 errors on Thu Jun 2 21:25:56 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 draid2:16d:20c:2s-0 DEGRADED 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 spare-9 DEGRADED 0 0 689 /root/test/test/files/file10 DEGRADED 0 0 187 too many errors draid2-0-0 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 INUSE currently in use draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# zpool clear pool-oss0 [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE scan: scrub repaired 0B in 00:00:30 with 0 errors on Thu Jun 2 21:26:26 2022 scan: resilvered (draid2:16d:20c:2s-0) 688M in 00:00:25 with 0 errors on Thu Jun 2 21:25:56 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 1 (f=1): [_(2),R(1),_(3)][98.1%][r=129MiB/s][r=128 IOPS][eta 00m:01s] test1: (groupid=0, jobs=6): err= 0: pid=39304: Thu Jun 2 21:28:34 2022 read: IOPS=229, BW=230MiB/s (241MB/s)(12.0GiB/53528msec) slat (usec): min=112, max=743753, avg=25253.22, stdev=66774.06 clat (usec): min=4, max=746417, avg=75813.29, stdev=99132.91 lat (usec): min=189, max=747405, avg=101073.44, stdev=103585.58 clat percentiles (usec): | 1.00th=[ 725], 5.00th=[ 1172], 10.00th=[ 1565], 20.00th=[ 2737], | 30.00th=[ 13042], 40.00th=[ 22676], 50.00th=[ 34866], 60.00th=[ 51119], | 70.00th=[ 77071], 80.00th=[131597], 90.00th=[227541], 95.00th=[295699], | 99.00th=[417334], 99.50th=[471860], 99.90th=[557843], 99.95th=[599786], | 99.99th=[742392] bw ( KiB/s): min=45799, max=1094478, per=100.00%, avg=241656.79, stdev=39434.54, samples=606 iops : min= 42, max= 1065, avg=232.11, stdev=38.60, samples=606 lat (usec) : 10=0.03%, 20=0.02%, 250=0.01%, 500=0.04%, 750=1.04% lat (usec) : 1000=1.95% lat (msec) : 2=11.85%, 4=7.66%, 10=4.52%, 20=10.25%, 50=22.14% lat (msec) : 100=15.84%, 250=16.57%, 500=7.76%, 750=0.33% cpu : usr=0.11%, sys=2.66%, ctx=5190, majf=13, minf=3162 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=230MiB/s (241MB/s), 230MiB/s-230MiB/s (241MB/s-241MB/s), io=12.0GiB (12.9GB), run=53528-53528msec [root@localhost files]# [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# zpool status -v pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an unrecoverable error. An attempt was made to correct the error. Applications are unaffected. action: Determine if the device needs to be replaced, and clear the errors using 'zpool clear' or replace the device with 'zpool replace'. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-9P scan: scrub repaired 0B in 00:00:21 with 0 errors on Thu Jun 2 21:28:47 2022 scan: resilvered (draid2:16d:20c:2s-0) 688M in 00:00:38 with 0 errors on Thu Jun 2 21:28:26 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 draid2:16d:20c:2s-0 DEGRADED 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 spare-9 DEGRADED 0 0 949 /root/test/test/files/file10 DEGRADED 0 0 235 too many errors draid2-0-0 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 INUSE currently in use draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# [root@localhost files]# ======================== RAIDZ2 ======================== + truncate -s 1G file1 file2 file3 file4 file5 file6 file7 file8 file9 file10 file11 file12 file13 file14 file15 file16 file17 file18 file19 file20 ++ pwd + p=/root/test/test/files + echo /root/test/test/files /root/test/test/files + zpool create -f -o cachefile=none -o failmode=panic -O canmount=off pool-oss0 raidz2 /root/test/test/files/file1 /root/test/test/files/file2 /root/test/test/files/file3 /root/test/test/files/file4 /root/test/test/files/file5 /root/test/test/files/file6 /root/test/test/files/file7 /root/test/test/files/file8 /root/test/test/files/file9 /root/test/test/files/file10 /root/test/test/files/file11 /root/test/test/files/file12 /root/test/test/files/file13 /root/test/test/files/file14 /root/test/test/files/file15 /root/test/test/files/file16 /root/test/test/files/file17 /root/test/test/files/file18 spare /root/test/test/files/file19 /root/test/test/files/file20 + zfs create -o mountpoint=/mnt/ost0 pool-oss0/ost0 + zfs set recordsize=4M pool-oss0 [root@localhost files]# [root@localhost files]# zpool status pool: pool-oss0 state: ONLINE config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 raidz2-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 spares /root/test/test/files/file19 AVAIL /root/test/test/files/file20 AVAIL errors: No known data errors [root@localhost files]# [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=write --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) Jobs: 5 (f=3): [W(2),f(1),W(1),f(1),_(1)][100.0%][w=266MiB/s][w=266 IOPS][eta 00m:00s] test1: (groupid=0, jobs=6): err= 0: pid=51354: Thu Jun 2 21:34:08 2022 write: IOPS=299, BW=299MiB/s (314MB/s)(12.0GiB/41059msec); 0 zone resets slat (usec): min=157, max=946620, avg=19745.63, stdev=94444.22 clat (usec): min=2, max=1358.4k, avg=59397.43, stdev=163201.02 lat (usec): min=172, max=1894.7k, avg=79164.52, stdev=186824.10 clat percentiles (msec): | 1.00th=[ 3], 5.00th=[ 4], 10.00th=[ 4], 20.00th=[ 6], | 30.00th=[ 6], 40.00th=[ 7], 50.00th=[ 8], 60.00th=[ 9], | 70.00th=[ 10], 80.00th=[ 11], 90.00th=[ 124], 95.00th=[ 514], | 99.00th=[ 743], 99.50th=[ 852], 99.90th=[ 953], 99.95th=[ 1099], | 99.99th=[ 1217] bw ( KiB/s): min=12081, max=1348858, per=100.00%, avg=338056.74, stdev=42775.70, samples=430 iops : min= 6, max= 1315, avg=326.61, stdev=41.88, samples=430 lat (usec) : 4=0.02%, 10=0.01%, 20=0.02%, 50=0.01%, 250=0.01% lat (usec) : 500=0.02%, 750=0.01%, 1000=0.02% lat (msec) : 2=0.49%, 4=10.19%, 10=67.02%, 20=10.28%, 50=1.29% lat (msec) : 100=0.45%, 250=0.84%, 500=3.61%, 750=4.91%, 1000=0.71% lat (msec) : 2000=0.10% cpu : usr=1.17%, sys=9.31%, ctx=2863, majf=0, minf=88 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=0,12288,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): WRITE: bw=299MiB/s (314MB/s), 299MiB/s-299MiB/s (314MB/s-314MB/s), io=12.0GiB (12.9GB), run=41059-41059msec [root@localhost files]# [root@localhost files]# md5sum /mnt/ost0/* > md5sum [root@localhost files]# [root@localhost files]# [root@localhost files]# dd if=/dev/zero of=file10 bs=1M count=1024 oflag=direct 1024+0 records in 1024+0 records out 1073741824 bytes (1.1 GB, 1.0 GiB) copied, 0.626179 s, 1.7 GB/s [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 2 (f=2): [_(2),R(1),_(2),R(1)][90.3%][r=297MiB/s][r=296 IOPS][eta 00m:06s] test1: (groupid=0, jobs=6): err= 0: pid=60803: Thu Jun 2 21:36:21 2022 read: IOPS=217, BW=218MiB/s (228MB/s)(12.0GiB/56491msec) slat (usec): min=112, max=1077.9k, avg=26250.57, stdev=73860.05 clat (usec): min=3, max=1079.3k, avg=78741.70, stdev=111960.22 lat (usec): min=177, max=1080.1k, avg=104999.01, stdev=118977.26 clat percentiles (usec): | 1.00th=[ 848], 5.00th=[ 1385], 10.00th=[ 1778], | 20.00th=[ 2868], 30.00th=[ 11731], 40.00th=[ 26346], | 50.00th=[ 47449], 60.00th=[ 68682], 70.00th=[ 91751], | 80.00th=[ 120062], 90.00th=[ 179307], 95.00th=[ 270533], | 99.00th=[ 534774], 99.50th=[ 683672], 99.90th=[1002439], | 99.95th=[1027605], 99.99th=[1082131] bw ( KiB/s): min=45966, max=1245756, per=100.00%, avg=237454.66, stdev=35048.49, samples=616 iops : min= 42, max= 1215, avg=228.27, stdev=34.30, samples=616 lat (usec) : 4=0.01%, 10=0.02%, 20=0.01%, 50=0.01%, 250=0.01% lat (usec) : 500=0.05%, 750=0.48%, 1000=1.21% lat (msec) : 2=11.26%, 4=9.36%, 10=6.04%, 20=7.89%, 50=14.62% lat (msec) : 100=22.80%, 250=20.61%, 500=4.12%, 750=1.10%, 1000=0.25% lat (msec) : 2000=0.15% cpu : usr=0.12%, sys=2.67%, ctx=5351, majf=1, minf=3175 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=218MiB/s (228MB/s), 218MiB/s-218MiB/s (228MB/s-228MB/s), io=12.0GiB (12.9GB), run=56491-56491msec [root@localhost files]# [root@localhost files]# md5sum -c md5sum /mnt/ost0/test1.0.0: OK /mnt/ost0/test1.1.0: OK /mnt/ost0/test1.2.0: OK /mnt/ost0/test1.3.0: OK /mnt/ost0/test1.4.0: OK /mnt/ost0/test1.5.0: OK [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an unrecoverable error. An attempt was made to correct the error. Applications are unaffected. action: Determine if the device needs to be replaced, and clear the errors using 'zpool clear' or replace the device with 'zpool replace'. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-9P scan: resilvered 1.42G in 00:00:52 with 0 errors on Thu Jun 2 21:36:21 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 raidz2-0 DEGRADED 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 spare-9 DEGRADED 0 0 2.96K /root/test/test/files/file10 DEGRADED 0 0 125 too many errors /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 spares /root/test/test/files/file19 INUSE currently in use /root/test/test/files/file20 AVAIL errors: No known data errors [root@localhost files]# [root@localhost files]# zpool clear pool-oss0 [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE scan: resilvered 1.42G in 00:00:52 with 0 errors on Thu Jun 2 21:36:21 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 raidz2-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 spares /root/test/test/files/file19 AVAIL /root/test/test/files/file20 AVAIL errors: No known data errors [root@localhost files]# [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 6 (f=6): [R(6)][94.7%][r=615MiB/s][r=614 IOPS][eta 00m:01s] test1: (groupid=0, jobs=6): err= 0: pid=76121: Thu Jun 2 21:37:57 2022 read: IOPS=686, BW=687MiB/s (720MB/s)(12.0GiB/17895msec) slat (usec): min=109, max=163406, avg=8609.37, stdev=17773.58 clat (usec): min=4, max=164918, avg=25852.37, stdev=23674.11 lat (usec): min=144, max=165676, avg=34466.52, stdev=22117.46 clat percentiles (usec): | 1.00th=[ 635], 5.00th=[ 955], 10.00th=[ 1319], 20.00th=[ 2245], | 30.00th=[ 8225], 40.00th=[ 14091], 50.00th=[ 20841], 60.00th=[ 28181], | 70.00th=[ 35390], 80.00th=[ 44303], 90.00th=[ 58983], 95.00th=[ 69731], | 99.00th=[ 99091], 99.50th=[110625], 99.90th=[143655], 99.95th=[149947], | 99.99th=[164627] bw ( KiB/s): min=393880, max=1161847, per=100.00%, avg=708601.84, stdev=30088.86, samples=207 iops : min= 382, max= 1134, avg=689.37, stdev=29.45, samples=207 lat (usec) : 10=0.05%, 250=0.02%, 500=0.13%, 750=2.04%, 1000=3.42% lat (msec) : 2=12.13%, 4=6.39%, 10=8.70%, 20=15.89%, 50=35.74% lat (msec) : 100=14.58%, 250=0.91% cpu : usr=0.23%, sys=6.75%, ctx=4852, majf=0, minf=3177 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=687MiB/s (720MB/s), 687MiB/s-687MiB/s (720MB/s-720MB/s), io=12.0GiB (12.9GB), run=17895-17895msec [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE scan: resilvered 1.42G in 00:00:52 with 0 errors on Thu Jun 2 21:36:21 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 raidz2-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 spares /root/test/test/files/file19 AVAIL /root/test/test/files/file20 AVAIL errors: No known data errors [root@localhost files]# [root@localhost files]# md5sum -c md5sum /mnt/ost0/test1.0.0: OK /mnt/ost0/test1.1.0: OK /mnt/ost0/test1.2.0: OK /mnt/ost0/test1.3.0: OK /mnt/ost0/test1.4.0: OK /mnt/ost0/test1.5.0: OK [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE scan: resilvered 1.42G in 00:00:52 with 0 errors on Thu Jun 2 21:36:21 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 raidz2-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 spares /root/test/test/files/file19 AVAIL /root/test/test/files/file20 AVAIL errors: No known data errors [root@localhost files]#