CASE 1: ======= + truncate -s 1G file1 file2 file3 file4 file5 file6 file7 file8 file9 file10 file11 file12 file13 file14 file15 file16 file17 file18 file19 file20 + pwd /root/test/test/files + zpool create -f -o cachefile=none -o failmode=panic -o multihost=on -O canmount=off pool-oss0 draid2:16d:2s /root/test/test/files/file1 /root/test/test/files/file2 /root/test/test/files/file3 /root/test/test/files/file4 /root/test/test/files/file5 /root/test/test/files/file6 /root/test/test/files/file7 /root/test/test/files/file8 /root/test/test/files/file9 /root/test/test/files/file10 /root/test/test/files/file11 /root/test/test/files/file12 /root/test/test/files/file13 /root/test/test/files/file14 /root/test/test/files/file15 /root/test/test/files/file16 /root/test/test/files/file17 /root/test/test/files/file18 /root/test/test/files/file19 /root/test/test/files/file20 + zfs create -o mountpoint=/mnt/ost0 pool-oss0/ost0 + zfs set recordsize=4M pool-oss0 [root@localhost ~]# zpool status -v pool: pool-oss0 state: ONLINE config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=write --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) Jobs: 3 (f=3): [_(1),W(2),_(2),W(1)][96.6%][w=224MiB/s][w=223 IOPS][eta 00m:02s] test1: (groupid=0, jobs=6): err= 0: pid=5169: Wed Jun 15 05:22:01 2022 write: IOPS=215, BW=216MiB/s (226MB/s)(12.0GiB/57015msec); 0 zone resets slat (usec): min=137, max=2314.5k, avg=26855.17, stdev=136419.26 clat (usec): min=2, max=4334.4k, avg=80821.23, stdev=241906.36 lat (usec): min=141, max=4341.6k, avg=107686.75, stdev=278562.89 clat percentiles (usec): | 1.00th=[ 1696], 5.00th=[ 2835], 10.00th=[ 3490], | 20.00th=[ 4555], 30.00th=[ 5538], 40.00th=[ 6521], | 50.00th=[ 7570], 60.00th=[ 8717], 70.00th=[ 10159], | 80.00th=[ 15926], 90.00th=[ 141558], 95.00th=[ 650118], | 99.00th=[ 884999], 99.50th=[1635779], 99.90th=[2122318], | 99.95th=[2298479], 99.99th=[4328522] bw ( KiB/s): min=10789, max=1254534, per=100.00%, avg=290484.43, stdev=38107.77, samples=506 iops : min= 6, max= 1223, avg=279.77, stdev=37.29, samples=506 lat (usec) : 4=0.02%, 10=0.01%, 20=0.01%, 50=0.01%, 250=0.01% lat (usec) : 500=0.07%, 750=0.09%, 1000=0.31% lat (msec) : 2=0.97%, 4=12.91%, 10=54.83%, 20=12.62%, 50=5.57% lat (msec) : 100=2.04%, 250=1.02%, 500=0.71%, 750=6.47%, 1000=1.62% lat (msec) : 2000=0.53%, >=2000=0.19% cpu : usr=0.91%, sys=7.19%, ctx=3089, majf=0, minf=94 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=0,12288,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): WRITE: bw=216MiB/s (226MB/s), 216MiB/s-216MiB/s (226MB/s-226MB/s), io=12.0GiB (12.9GB), run=57015-57015msec [root@localhost files]# md5sum /mnt/ost0/* > md5sum [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 3 (f=3): [_(1),R(3),_(2)][90.0%][r=562MiB/s][r=561 IOPS][eta 00m:02s] test1: (groupid=0, jobs=6): err= 0: pid=18093: Wed Jun 15 05:24:32 2022 read: IOPS=691, BW=692MiB/s (725MB/s)(12.0GiB/17766msec) slat (usec): min=99, max=157459, avg=8096.96, stdev=17329.60 clat (usec): min=3, max=159256, avg=24312.20, stdev=23748.71 lat (usec): min=149, max=159906, avg=32415.76, stdev=22985.40 clat percentiles (usec): | 1.00th=[ 668], 5.00th=[ 1029], 10.00th=[ 1418], 20.00th=[ 2343], | 30.00th=[ 7570], 40.00th=[ 12518], 50.00th=[ 17957], 60.00th=[ 24511], | 70.00th=[ 31851], 80.00th=[ 41681], 90.00th=[ 56886], 95.00th=[ 71828], | 99.00th=[103285], 99.50th=[114820], 99.90th=[143655], 99.95th=[145753], | 99.99th=[158335] bw ( KiB/s): min=423240, max=1347530, per=100.00%, avg=753898.23, stdev=33852.34, samples=196 iops : min= 410, max= 1314, avg=733.82, stdev=33.10, samples=196 lat (usec) : 4=0.02%, 10=0.02%, 20=0.01%, 250=0.02%, 500=0.14% lat (usec) : 750=1.67%, 1000=2.73% lat (msec) : 2=12.37%, 4=7.90%, 10=9.67%, 20=19.14%, 50=32.96% lat (msec) : 100=12.22%, 250=1.14% cpu : usr=0.26%, sys=7.23%, ctx=5076, majf=0, minf=6222 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=692MiB/s (725MB/s), 692MiB/s-692MiB/s (725MB/s-725MB/s), io=12.0GiB (12.9GB), run=17766-17766msec [root@localhost files]# [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 1 (f=1): [_(2),R(1),_(3)][98.0%][r=159MiB/s][r=158 IOPS][eta 00m:01s] test1: (groupid=0, jobs=6): err= 0: pid=21965: Wed Jun 15 05:26:23 2022 read: IOPS=251, BW=252MiB/s (264MB/s)(12.0GiB/48788msec) slat (usec): min=102, max=669498, avg=22678.70, stdev=56395.12 clat (usec): min=3, max=673674, avg=67996.21, stdev=81885.07 lat (usec): min=255, max=673938, avg=90683.10, stdev=83967.64 clat percentiles (usec): | 1.00th=[ 783], 5.00th=[ 1254], 10.00th=[ 1745], 20.00th=[ 3228], | 30.00th=[ 15401], 40.00th=[ 26870], 50.00th=[ 39060], 60.00th=[ 54264], | 70.00th=[ 73925], 80.00th=[113771], 90.00th=[187696], 95.00th=[248513], | 99.00th=[354419], 99.50th=[387974], 99.90th=[450888], 99.95th=[467665], | 99.99th=[675283] bw ( KiB/s): min=48363, max=880077, per=100.00%, avg=270074.34, stdev=35194.88, samples=546 iops : min= 42, max= 856, avg=259.15, stdev=34.45, samples=546 lat (usec) : 4=0.01%, 10=0.03%, 20=0.01%, 500=0.06%, 750=0.70% lat (usec) : 1000=1.75% lat (msec) : 2=10.42%, 4=8.62%, 10=4.96%, 20=7.10%, 50=24.35% lat (msec) : 100=19.21%, 250=17.79%, 500=4.93%, 750=0.05% cpu : usr=0.11%, sys=3.21%, ctx=5667, majf=0, minf=6238 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=252MiB/s (264MB/s), 252MiB/s-252MiB/s (264MB/s-264MB/s), io=12.0GiB (12.9GB), run=48788-48788msec [root@localhost files]# [root@localhost files]# md5sum -c md5sum /mnt/ost0/test1.0.0: OK /mnt/ost0/test1.1.0: OK /mnt/ost0/test1.2.0: OK /mnt/ost0/test1.3.0: OK /mnt/ost0/test1.4.0: OK /mnt/ost0/test1.5.0: OK [root@localhost files]# zpool status -v pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an unrecoverable error. An attempt was made to correct the error. Applications are unaffected. action: Determine if the device needs to be replaced, and clear the errors using 'zpool clear' or replace the device with 'zpool replace'. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-9P scan: scrub repaired 1.26G in 00:00:32 with 0 errors on Wed Jun 15 05:26:39 2022 scan: resilvered (draid2:16d:20c:2s-0) 690M in 00:00:28 with 0 errors on Wed Jun 15 05:26:07 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 draid2:16d:20c:2s-0 DEGRADED 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 spare-9 DEGRADED 0 0 2.56K /root/test/test/files/file10 DEGRADED 0 0 234 too many errors draid2-0-0 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 INUSE currently in use draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# zpool clear pool-oss0 [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE scan: scrub repaired 1.26G in 00:00:32 with 0 errors on Wed Jun 15 05:26:39 2022 scan: resilvered (draid2:16d:20c:2s-0) 690M in 00:00:28 with 0 errors on Wed Jun 15 05:26:07 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 2 (f=2): [_(1),R(2),_(3)][95.0%][r=582MiB/s][r=581 IOPS][eta 00m:01s] test1: (groupid=0, jobs=6): err= 0: pid=40616: Wed Jun 15 05:28:56 2022 read: IOPS=672, BW=673MiB/s (705MB/s)(12.0GiB/18266msec) slat (usec): min=106, max=165055, avg=8409.73, stdev=17306.40 clat (usec): min=3, max=167233, avg=25268.72, stdev=23637.03 lat (usec): min=124, max=170302, avg=33684.46, stdev=22779.38 clat percentiles (usec): | 1.00th=[ 775], 5.00th=[ 1254], 10.00th=[ 1745], 20.00th=[ 3425], | 30.00th=[ 9110], 40.00th=[ 13829], 50.00th=[ 19006], 60.00th=[ 25035], | 70.00th=[ 32900], 80.00th=[ 42730], 90.00th=[ 58459], 95.00th=[ 71828], | 99.00th=[102237], 99.50th=[115868], 99.90th=[135267], 99.95th=[137364], | 99.99th=[166724] bw ( KiB/s): min=410058, max=1220605, per=100.00%, avg=722792.09, stdev=32849.53, samples=201 iops : min= 397, max= 1189, avg=702.48, stdev=32.13, samples=201 lat (usec) : 4=0.01%, 10=0.02%, 20=0.02%, 250=0.02%, 500=0.11% lat (usec) : 750=0.68%, 1000=1.71% lat (msec) : 2=9.99%, 4=8.94%, 10=10.32%, 20=20.20%, 50=33.41% lat (msec) : 100=13.48%, 250=1.09% cpu : usr=0.26%, sys=9.06%, ctx=5503, majf=0, minf=5716 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=673MiB/s (705MB/s), 673MiB/s-673MiB/s (705MB/s-705MB/s), io=12.0GiB (12.9GB), run=18266-18266msec [root@localhost files]# md5sum -c md5sum /mnt/ost0/test1.0.0: OK /mnt/ost0/test1.1.0: OK /mnt/ost0/test1.2.0: OK /mnt/ost0/test1.3.0: OK /mnt/ost0/test1.4.0: OK /mnt/ost0/test1.5.0: OK [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE scan: scrub repaired 1.26G in 00:00:32 with 0 errors on Wed Jun 15 05:26:39 2022 scan: resilvered (draid2:16d:20c:2s-0) 690M in 00:00:28 with 0 errors on Wed Jun 15 05:26:07 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors ==END OF CASE1============================= CASE 2: ======= + truncate -s 1G file1 file2 file3 file4 file5 file6 file7 file8 file9 file10 file11 file12 file13 file14 file15 file16 file17 file18 file19 file20 + pwd /root/test/test/files + zpool create -f -o cachefile=none -o failmode=panic -o multihost=on -O canmount=off pool-oss0 draid2:16d:2s /root/test/test/files/file1 /root/test/test/files/file2 /root/test/test/files/file3 /root/test/test/files/file4 /root/test/test/files/file5 /root/test/test/files/file6 /root/test/test/files/file7 /root/test/test/files/file8 /root/test/test/files/file9 /root/test/test/files/file10 /root/test/test/files/file11 /root/test/test/files/file12 /root/test/test/files/file13 /root/test/test/files/file14 /root/test/test/files/file15 /root/test/test/files/file16 /root/test/test/files/file17 /root/test/test/files/file18 /root/test/test/files/file19 /root/test/test/files/file20 + zfs create -o mountpoint=/mnt/ost0 pool-oss0/ost0 + zfs set recordsize=4M pool-oss0 [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=write --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) Jobs: 4 (f=4): [_(1),W(3),_(1),W(1)][96.2%][w=228MiB/s][w=227 IOPS][eta 00m:02s] test1: (groupid=0, jobs=6): err= 0: pid=78422: Wed Jun 15 05:50:08 2022 write: IOPS=241, BW=242MiB/s (254MB/s)(12.0GiB/50796msec); 0 zone resets slat (usec): min=155, max=1131.0k, avg=23827.76, stdev=114404.95 clat (usec): min=4, max=1929.2k, avg=71809.71, stdev=202312.07 lat (usec): min=167, max=1930.5k, avg=95655.09, stdev=232925.59 clat percentiles (usec): | 1.00th=[ 1991], 5.00th=[ 2966], 10.00th=[ 3621], | 20.00th=[ 4686], 30.00th=[ 5604], 40.00th=[ 6456], | 50.00th=[ 7373], 60.00th=[ 8225], 70.00th=[ 9241], | 80.00th=[ 10683], 90.00th=[ 120062], 95.00th=[ 624952], | 99.00th=[ 901776], 99.50th=[ 926942], 99.90th=[1451230], | 99.95th=[1736442], 99.99th=[1904215] bw ( KiB/s): min=11297, max=1130998, per=100.00%, avg=313115.20, stdev=38025.08, samples=460 iops : min= 6, max= 1101, avg=301.91, stdev=37.21, samples=460 lat (usec) : 10=0.02%, 20=0.02%, 50=0.01%, 250=0.01%, 500=0.02% lat (usec) : 750=0.03%, 1000=0.09% lat (msec) : 2=0.82%, 4=12.48%, 10=62.91%, 20=10.80%, 50=2.18% lat (msec) : 100=0.50%, 250=0.40%, 500=1.03%, 750=6.72%, 1000=1.65% lat (msec) : 2000=0.31% cpu : usr=0.92%, sys=7.41%, ctx=2540, majf=0, minf=79 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=0,12288,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): WRITE: bw=242MiB/s (254MB/s), 242MiB/s-242MiB/s (254MB/s-254MB/s), io=12.0GiB (12.9GB), run=50796-50796msec [root@localhost files]# md5sum /mnt/ost0/* > md5sum [root@localhost files]# dd if=/dev/zero of=file10 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.606859 s, 1.8 GB/s [root@localhost files]# dd if=/dev/zero of=file11 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.619531 s, 1.7 GB/s [root@localhost files]# [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 1 (f=0): [_(4),f(1),_(1)][100.0%][r=102MiB/s][r=102 IOPS][eta 00m:00s] test1: (groupid=0, jobs=6): err= 0: pid=94954: Wed Jun 15 05:54:30 2022 read: IOPS=116, BW=117MiB/s (123MB/s)(12.0GiB/105060msec) slat (usec): min=121, max=1090.1k, avg=50698.79, stdev=140945.06 clat (usec): min=3, max=1092.1k, avg=152138.59, stdev=212404.43 lat (usec): min=162, max=1093.1k, avg=202844.29, stdev=224578.41 clat percentiles (usec): | 1.00th=[ 848], 5.00th=[ 1369], 10.00th=[ 1811], | 20.00th=[ 3130], 30.00th=[ 16319], 40.00th=[ 29754], | 50.00th=[ 45351], 60.00th=[ 65274], 70.00th=[ 111674], | 80.00th=[ 362808], 90.00th=[ 513803], 95.00th=[ 608175], | 99.00th=[ 775947], 99.50th=[ 859833], 99.90th=[ 960496], | 99.95th=[ 977273], 99.99th=[1098908] bw ( KiB/s): min=27649, max=876988, per=100.00%, avg=131749.48, stdev=29362.89, samples=1106 iops : min= 24, max= 853, avg=124.56, stdev=28.72, samples=1106 lat (usec) : 4=0.01%, 10=0.02%, 20=0.02%, 250=0.01%, 500=0.03% lat (usec) : 750=0.45%, 1000=1.34% lat (msec) : 2=10.63%, 4=9.33%, 10=4.05%, 20=7.06%, 50=19.80% lat (msec) : 100=15.91%, 250=6.46%, 500=13.97%, 750=9.64%, 1000=1.23% lat (msec) : 2000=0.05% cpu : usr=0.05%, sys=1.50%, ctx=5315, majf=10, minf=6238 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=117MiB/s (123MB/s), 117MiB/s-117MiB/s (123MB/s-123MB/s), io=12.0GiB (12.9GB), run=105060-105060msec [root@localhost files]# md5sum -c md5sum /mnt/ost0/test1.0.0: OK /mnt/ost0/test1.1.0: OK /mnt/ost0/test1.2.0: OK /mnt/ost0/test1.3.0: OK /mnt/ost0/test1.4.0: OK /mnt/ost0/test1.5.0: OK [root@localhost files]# [root@localhost files]# zpool status -v 2 pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an error resulting in data corruption. Applications may be affected. action: Restore the file in question if possible. Otherwise restore the entire pool from backup. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-8A scan: scrub repaired 2.54G in 00:00:36 with 3 errors on Wed Jun 15 05:54:49 2022 scan: resilvered (draid2:16d:20c:2s-0) 1.35G in 00:00:53 with 0 errors on Wed Jun 15 05:54:13 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 draid2:16d:20c:2s-0 DEGRADED 0 0 0 /root/test/test/files/file1 ONLINE 0 0 10 /root/test/test/files/file2 ONLINE 0 0 12 /root/test/test/files/file3 ONLINE 0 0 12 /root/test/test/files/file4 ONLINE 0 0 10 /root/test/test/files/file5 ONLINE 0 0 12 /root/test/test/files/file6 ONLINE 0 0 10 /root/test/test/files/file7 ONLINE 0 0 8 /root/test/test/files/file8 ONLINE 0 0 12 /root/test/test/files/file9 ONLINE 0 0 10 spare-9 DEGRADED 0 0 2.67K /root/test/test/files/file10 DEGRADED 0 0 185 too many errors draid2-0-0 ONLINE 0 0 0 spare-10 DEGRADED 0 0 2.46K /root/test/test/files/file11 DEGRADED 0 0 405 too many errors draid2-0-1 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 10 /root/test/test/files/file13 ONLINE 0 0 12 /root/test/test/files/file14 ONLINE 0 0 10 /root/test/test/files/file15 ONLINE 0 0 10 /root/test/test/files/file16 ONLINE 0 0 12 /root/test/test/files/file17 ONLINE 0 0 12 /root/test/test/files/file18 ONLINE 0 0 12 /root/test/test/files/file19 ONLINE 0 0 12 /root/test/test/files/file20 ONLINE 0 0 12 spares draid2-0-0 INUSE currently in use draid2-0-1 INUSE currently in use errors: Permanent errors have been detected in the following files: /mnt/ost0/test1.0.0 /mnt/ost0/test1.2.0 pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an error resulting in data corruption. Applications may be affected. action: Restore the file in question if possible. Otherwise restore the entire pool from backup. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-8A scan: scrub repaired 2.54G in 00:00:36 with 3 errors on Wed Jun 15 05:54:49 2022 scan: resilvered (draid2:16d:20c:2s-0) 1.35G in 00:00:53 with 0 errors on Wed Jun 15 05:54:13 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 draid2:16d:20c:2s-0 DEGRADED 0 0 0 /root/test/test/files/file1 ONLINE 0 0 10 /root/test/test/files/file2 ONLINE 0 0 12 /root/test/test/files/file3 ONLINE 0 0 12 /root/test/test/files/file4 ONLINE 0 0 10 /root/test/test/files/file5 ONLINE 0 0 12 /root/test/test/files/file6 ONLINE 0 0 10 /root/test/test/files/file7 ONLINE 0 0 8 /root/test/test/files/file8 ONLINE 0 0 12 /root/test/test/files/file9 ONLINE 0 0 10 spare-9 DEGRADED 0 0 2.67K /root/test/test/files/file10 DEGRADED 0 0 185 too many errors draid2-0-0 ONLINE 0 0 0 spare-10 DEGRADED 0 0 2.46K /root/test/test/files/file11 DEGRADED 0 0 405 too many errors draid2-0-1 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 10 /root/test/test/files/file13 ONLINE 0 0 12 /root/test/test/files/file14 ONLINE 0 0 10 /root/test/test/files/file15 ONLINE 0 0 10 /root/test/test/files/file16 ONLINE 0 0 12 /root/test/test/files/file17 ONLINE 0 0 12 /root/test/test/files/file18 ONLINE 0 0 12 /root/test/test/files/file19 ONLINE 0 0 12 /root/test/test/files/file20 ONLINE 0 0 12 spares draid2-0-0 INUSE currently in use draid2-0-1 INUSE currently in use munmap_chunk(): invalid pointer Aborted (core dumped) [root@localhost files]# [root@localhost files]# zpool clear pool-oss0 [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE status: One or more devices has experienced an error resulting in data corruption. Applications may be affected. action: Restore the file in question if possible. Otherwise restore the entire pool from backup. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-8A scan: scrub repaired 2.54G in 00:00:36 with 3 errors on Wed Jun 15 05:54:49 2022 scan: resilvered (draid2:16d:20c:2s-0) 1.35G in 00:00:53 with 0 errors on Wed Jun 15 05:54:13 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: Permanent errors have been detected in the following files: /mnt/ost0/test1.0.0 /mnt/ost0/test1.2.0 [root@localhost files]# [root@localhost files]# zpool scrub pool-oss0 [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE status: One or more devices has experienced an unrecoverable error. An attempt was made to correct the error. Applications are unaffected. action: Determine if the device needs to be replaced, and clear the errors using 'zpool clear' or replace the device with 'zpool replace'. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-9P scan: scrub repaired 64.1M in 00:00:18 with 0 errors on Wed Jun 15 05:57:26 2022 scan: resilvered (draid2:16d:20c:2s-0) 1.35G in 00:00:53 with 0 errors on Wed Jun 15 05:54:13 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 155 /root/test/test/files/file11 ONLINE 0 0 129 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# zpool clear pool-oss0 [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE scan: scrub repaired 64.1M in 00:00:18 with 0 errors on Wed Jun 15 05:57:26 2022 scan: resilvered (draid2:16d:20c:2s-0) 1.35G in 00:00:53 with 0 errors on Wed Jun 15 05:54:13 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# md5sum -c md5sum /mnt/ost0/test1.0.0: OK /mnt/ost0/test1.1.0: OK /mnt/ost0/test1.2.0: OK /mnt/ost0/test1.3.0: OK /mnt/ost0/test1.4.0: OK /mnt/ost0/test1.5.0: OK [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE scan: scrub repaired 64.1M in 00:00:18 with 0 errors on Wed Jun 15 05:57:26 2022 scan: resilvered (draid2:16d:20c:2s-0) 1.35G in 00:00:53 with 0 errors on Wed Jun 15 05:54:13 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors ==END OF CASE2============================= CASE 3: ====== + truncate -s 1G file1 file2 file3 file4 file5 file6 file7 file8 file9 file10 file11 file12 file13 file14 file15 file16 file17 file18 file19 file20 + pwd /root/test/test/files + zpool create -f -o cachefile=none -o failmode=panic -o multihost=on -O canmount=off pool-oss0 draid2:16d:2s /root/test/test/files/file1 /root/test/test/files/file2 /root/test/test/files/file3 /root/test/test/files/file4 /root/test/test/files/file5 /root/test/test/files/file6 /root/test/test/files/file7 /root/test/test/files/file8 /root/test/test/files/file9 /root/test/test/files/file10 /root/test/test/files/file11 /root/test/test/files/file12 /root/test/test/files/file13 /root/test/test/files/file14 /root/test/test/files/file15 /root/test/test/files/file16 /root/test/test/files/file17 /root/test/test/files/file18 /root/test/test/files/file19 /root/test/test/files/file20 + zfs create -o mountpoint=/mnt/ost0 pool-oss0/ost0 + zfs set recordsize=4M pool-oss0 [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=write --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) test1: Laying out IO file (1 file / 2048MiB) Jobs: 1 (f=1): [_(4),W(1),_(1)][98.0%][w=236MiB/s][w=235 IOPS][eta 00m:01s] test1: (groupid=0, jobs=6): err= 0: pid=216814: Wed Jun 15 06:18:10 2022 write: IOPS=246, BW=246MiB/s (258MB/s)(12.0GiB/49867msec); 0 zone resets slat (usec): min=126, max=1286.1k, avg=23503.26, stdev=114906.98 clat (usec): min=2, max=1760.8k, avg=70653.20, stdev=201755.47 lat (usec): min=165, max=2304.5k, avg=94167.80, stdev=232152.71 clat percentiles (usec): | 1.00th=[ 930], 5.00th=[ 2376], 10.00th=[ 3294], | 20.00th=[ 4555], 30.00th=[ 5604], 40.00th=[ 6587], | 50.00th=[ 7504], 60.00th=[ 8356], 70.00th=[ 9241], | 80.00th=[ 10814], 90.00th=[ 85459], 95.00th=[ 633340], | 99.00th=[ 851444], 99.50th=[ 952108], 99.90th=[1468007], | 99.95th=[1518339], 99.99th=[1669333] bw ( KiB/s): min=11814, max=1690831, per=100.00%, avg=309365.31, stdev=47284.05, samples=471 iops : min= 6, max= 1649, avg=298.41, stdev=46.26, samples=471 lat (usec) : 4=0.02%, 10=0.01%, 20=0.02%, 50=0.01%, 250=0.01% lat (usec) : 500=0.11%, 750=0.29%, 1000=1.01% lat (msec) : 2=2.33%, 4=11.35%, 10=60.62%, 20=11.48%, 50=2.29% lat (msec) : 100=0.59%, 250=0.54%, 500=0.87%, 750=6.61%, 1000=1.46% lat (msec) : 2000=0.39% cpu : usr=0.92%, sys=7.45%, ctx=2934, majf=0, minf=85 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=0,12288,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): WRITE: bw=246MiB/s (258MB/s), 246MiB/s-246MiB/s (258MB/s-258MB/s), io=12.0GiB (12.9GB), run=49867-49867msec [root@localhost files]# [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 2 (f=2): [_(3),R(1),_(1),R(1)][95.0%][r=543MiB/s][r=542 IOPS][eta 00m:01s] test1: (groupid=0, jobs=6): err= 0: pid=224674: Wed Jun 15 06:18:37 2022 read: IOPS=671, BW=672MiB/s (704MB/s)(12.0GiB/18291msec) slat (usec): min=107, max=206684, avg=8400.81, stdev=17964.26 clat (usec): min=5, max=207906, avg=25236.68, stdev=24655.03 lat (usec): min=279, max=208337, avg=33643.37, stdev=23898.28 clat percentiles (usec): | 1.00th=[ 717], 5.00th=[ 1106], 10.00th=[ 1516], 20.00th=[ 2507], | 30.00th=[ 9372], 40.00th=[ 13960], 50.00th=[ 18482], 60.00th=[ 24511], | 70.00th=[ 32637], 80.00th=[ 42206], 90.00th=[ 57934], 95.00th=[ 71828], | 99.00th=[107480], 99.50th=[129500], 99.90th=[168821], 99.95th=[183501], | 99.99th=[208667] bw ( KiB/s): min=385151, max=1322628, per=100.00%, avg=723385.11, stdev=34345.15, samples=203 iops : min= 373, max= 1289, avg=703.54, stdev=33.58, samples=203 lat (usec) : 10=0.02%, 20=0.02%, 50=0.01%, 500=0.09%, 750=1.12% lat (usec) : 1000=2.50% lat (msec) : 2=12.35%, 4=7.54%, 10=7.84%, 20=21.22%, 50=33.11% lat (msec) : 100=12.61%, 250=1.58% cpu : usr=0.31%, sys=7.02%, ctx=4958, majf=0, minf=6228 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=672MiB/s (704MB/s), 672MiB/s-672MiB/s (704MB/s-704MB/s), io=12.0GiB (12.9GB), run=18291-18291msec [root@localhost files]# [root@localhost files]# md5sum /mnt/ost0/* > md5sum [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# [root@localhost files]# dd if=/dev/zero of=file1 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.638954 s, 1.7 GB/s [root@localhost files]# dd if=/dev/zero of=file2 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.5486 s, 1.9 GB/s [root@localhost files]# [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 1 (f=1): [_(4),R(1),_(1)][98.9%][r=27.8MiB/s][r=27 IOPS][eta 00m:01s] test1: (groupid=0, jobs=6): err= 0: pid=232717: Wed Jun 15 06:21:41 2022 read: IOPS=133, BW=134MiB/s (140MB/s)(12.0GiB/92031msec) slat (usec): min=133, max=699185, avg=42623.09, stdev=88703.67 clat (usec): min=6, max=724631, avg=127834.55, stdev=115217.72 lat (usec): min=402, max=727566, avg=170463.77, stdev=103945.84 clat percentiles (usec): | 1.00th=[ 922], 5.00th=[ 1369], 10.00th=[ 1778], 20.00th=[ 3458], | 30.00th=[ 33817], 40.00th=[ 79168], 50.00th=[114820], 60.00th=[147850], | 70.00th=[183501], 80.00th=[223347], 90.00th=[287310], 95.00th=[333448], | 99.00th=[446694], 99.50th=[513803], 99.90th=[641729], 99.95th=[666895], | 99.99th=[725615] bw ( KiB/s): min=45316, max=635595, per=100.00%, avg=144081.70, stdev=15773.50, samples=1018 iops : min= 40, max= 617, avg=136.57, stdev=15.45, samples=1018 lat (usec) : 10=0.03%, 20=0.02%, 500=0.02%, 750=0.31%, 1000=1.16% lat (msec) : 2=10.99%, 4=8.59%, 10=3.28%, 20=2.41%, 50=7.51% lat (msec) : 100=11.13%, 250=39.58%, 500=14.39%, 750=0.59% cpu : usr=0.07%, sys=1.71%, ctx=5289, majf=3, minf=4199 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=134MiB/s (140MB/s), 134MiB/s-134MiB/s (140MB/s-140MB/s), io=12.0GiB (12.9GB), run=92031-92031msec [root@localhost files]# [root@localhost files]# md5sum -c md5sum /mnt/ost0/test1.0.0: OK /mnt/ost0/test1.1.0: OK /mnt/ost0/test1.2.0: OK /mnt/ost0/test1.3.0: OK /mnt/ost0/test1.4.0: OK /mnt/ost0/test1.5.0: OK [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an unrecoverable error. An attempt was made to correct the error. Applications are unaffected. action: Determine if the device needs to be replaced, and clear the errors using 'zpool clear' or replace the device with 'zpool replace'. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-9P scan: scrub repaired 1.66G in 00:00:57 with 3 errors on Wed Jun 15 06:21:42 2022 scan: resilvered (draid2:16d:20c:2s-0) 689M in 00:00:36 with 0 errors on Wed Jun 15 06:20:45 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 draid2:16d:20c:2s-0 DEGRADED 0 0 0 spare-0 DEGRADED 0 0 2.65K /root/test/test/files/file1 DEGRADED 0 0 152 too many errors draid2-0-0 ONLINE 0 0 0 spare-1 DEGRADED 0 0 2.94K /root/test/test/files/file2 DEGRADED 0 0 1011 too many errors draid2-0-1 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 8 /root/test/test/files/file4 ONLINE 0 0 8 /root/test/test/files/file5 ONLINE 0 0 12 /root/test/test/files/file6 ONLINE 0 0 12 /root/test/test/files/file7 ONLINE 0 0 12 /root/test/test/files/file8 ONLINE 0 0 12 /root/test/test/files/file9 ONLINE 0 0 12 /root/test/test/files/file10 ONLINE 0 0 8 /root/test/test/files/file11 ONLINE 0 0 8 /root/test/test/files/file12 ONLINE 0 0 12 /root/test/test/files/file13 ONLINE 0 0 12 /root/test/test/files/file14 ONLINE 0 0 8 /root/test/test/files/file15 ONLINE 0 0 12 /root/test/test/files/file16 ONLINE 0 0 12 /root/test/test/files/file17 ONLINE 0 0 12 /root/test/test/files/file18 ONLINE 0 0 12 /root/test/test/files/file19 ONLINE 0 0 12 /root/test/test/files/file20 ONLINE 0 0 12 spares draid2-0-0 INUSE currently in use draid2-0-1 INUSE currently in use errors: No known data errors [root@localhost files]# [root@localhost files]# zpool clear pool-oss0 [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE scan: scrub repaired 1.66G in 00:00:57 with 3 errors on Wed Jun 15 06:21:42 2022 scan: resilvered (draid2:16d:20c:2s-0) 689M in 00:00:36 with 0 errors on Wed Jun 15 06:20:45 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# dd if=/dev/zero of=file10 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.561195 s, 1.9 GB/s [root@localhost files]# dd if=/dev/zero of=file11 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.574205 s, 1.9 GB/s [root@localhost files]# fio --name=test1 --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --directory=/mnt/ost0/ --group_reporting test1: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 6 (f=6): [R(6)][3.6%][r=219MiB/s][r=219 IOPS][eta 01m:21s] Message from syslogd@localhost at Jun 15 06:23:39 ... kernel:VERIFY3(vd->vdev_rebuild_reset_wanted == B_FALSE) failed (1 == 0) Message from syslogd@localhost at Jun 15 06:23:39 ... kernel:PANIC at vdev_rebuild.c:763:vdev_rebuild_thread() Jobs: 6 (f=6): [R(6)][39.9%][r=55.6MiB/s][r=55 IOPS][eta 01m:35s] pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an unrecoverable error. An attempt was made to correct the error. Applications are unaffected. action: Determine if the device needs to be replaced, and clear the errors using 'zpool clear' or replace the device with 'zpool replace'. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-9P scan: scrub repaired 1.66G in 00:00:57 with 3 errors on Wed Jun 15 06:21:42 2022 scan: resilvered (draid2:16d:20c:2s-0) 689M in 00:00:36 with 0 errors on Wed Jun 15 06:20:45 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 draid2:16d:20c:2s-0 DEGRADED 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 DEGRADED 0 0 110 too many errors /root/test/test/files/file11 ONLINE 0 0 22 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors Message from syslogd@localhost at Jun 15 06:23:39 ... kernel:VERIFY3(vd->vdev_rebuild_reset_wanted == B_FALSE) failed (1 == 0) Message from syslogd@localhost at Jun 15 06:23:39 ... kernel:PANIC at vdev_rebuild.c:763:vdev_rebuild_thread() dmesg: ----- [Wed Jun 15 06:23:38 2022] VERIFY3(vd->vdev_rebuild_reset_wanted == B_FALSE) failed (1 == 0) [Wed Jun 15 06:23:38 2022] PANIC at vdev_rebuild.c:763:vdev_rebuild_thread() [Wed Jun 15 06:23:38 2022] Showing stack for process 261796 [Wed Jun 15 06:23:38 2022] CPU: 2 PID: 261796 Comm: vdev_rebuild Kdump: loaded Tainted: P OE ---------r- - 4.18.0-305.10.2.x6.2.000.51.x86_64 #1 [Wed Jun 15 06:23:38 2022] Hardware name: innotek GmbH VirtualBox/VirtualBox, BIOS VirtualBox 12/01/2006 [Wed Jun 15 06:23:38 2022] Call Trace: [Wed Jun 15 06:23:38 2022] dump_stack+0x5c/0x80 [Wed Jun 15 06:23:38 2022] spl_panic+0xd3/0xfb [spl] [Wed Jun 15 06:23:38 2022] ? kvm_sched_clock_read+0xd/0x20 [Wed Jun 15 06:23:38 2022] ? __switch_to_asm+0x41/0x70 [Wed Jun 15 06:23:38 2022] ? __switch_to_asm+0x35/0x70 [Wed Jun 15 06:23:38 2022] ? __switch_to_asm+0x41/0x70 [Wed Jun 15 06:23:38 2022] ? __switch_to_asm+0x35/0x70 [Wed Jun 15 06:23:38 2022] ? __switch_to_asm+0x41/0x70 [Wed Jun 15 06:23:38 2022] ? __switch_to_asm+0x35/0x70 [Wed Jun 15 06:23:38 2022] ? __switch_to_asm+0x41/0x70 [Wed Jun 15 06:23:38 2022] ? _cond_resched+0x15/0x30 [Wed Jun 15 06:23:38 2022] ? mutex_lock+0xe/0x30 [Wed Jun 15 06:23:38 2022] ? vdev_rebuild_range+0xa50/0xa50 [zfs] [Wed Jun 15 06:23:38 2022] ? spl_taskq_fini+0x70/0x70 [spl] [Wed Jun 15 06:23:38 2022] vdev_rebuild_thread+0xbb6/0xdb0 [zfs] [Wed Jun 15 06:23:38 2022] ? thread_generic_wrapper+0x6b/0xb0 [spl] [Wed Jun 15 06:23:38 2022] ? vdev_rebuild_range+0xa50/0xa50 [zfs] [Wed Jun 15 06:23:38 2022] ? spl_taskq_fini+0x70/0x70 [spl] [Wed Jun 15 06:23:38 2022] thread_generic_wrapper+0x78/0xb0 [spl] [Wed Jun 15 06:23:38 2022] kthread+0x116/0x130 [Wed Jun 15 06:23:38 2022] ? kthread_flush_work_fn+0x10/0x10 [Wed Jun 15 06:23:38 2022] ret_from_fork+0x35/0x40 [Wed Jun 15 06:26:00 2022] INFO: task agents:1359 blocked for more than 120 seconds. [Wed Jun 15 06:26:00 2022] Tainted: P OE ---------r- - 4.18.0-305.10.2.x6.2.000.51.x86_64 #1 [Wed Jun 15 06:26:00 2022] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed Jun 15 06:26:00 2022] agents D 0 1359 1 0x00000080 [Wed Jun 15 06:26:00 2022] Call Trace: [Wed Jun 15 06:26:00 2022] __schedule+0x2c4/0x700 [Wed Jun 15 06:26:00 2022] schedule+0x38/0xa0 [Wed Jun 15 06:26:00 2022] schedule_preempt_disabled+0xa/0x10 [Wed Jun 15 06:26:00 2022] __mutex_lock.isra.6+0x2d0/0x4a0 [Wed Jun 15 06:26:00 2022] ? nvlist_xalloc.part.3+0x4b/0xb0 [zfs] [Wed Jun 15 06:26:00 2022] spa_all_configs+0x58/0x1f0 [zfs] [Wed Jun 15 06:26:00 2022] zfs_ioc_pool_configs+0x16/0x60 [zfs] [Wed Jun 15 06:26:00 2022] zfsdev_ioctl_common+0x656/0x980 [zfs] [Wed Jun 15 06:26:00 2022] zfsdev_ioctl+0x4f/0xe0 [zfs] [Wed Jun 15 06:26:00 2022] do_vfs_ioctl+0xa4/0x680 [Wed Jun 15 06:26:00 2022] ? handle_mm_fault+0xc2/0x1d0 [Wed Jun 15 06:26:00 2022] ? syscall_trace_enter+0x1d3/0x2c0 [Wed Jun 15 06:26:00 2022] ksys_ioctl+0x60/0x90 [Wed Jun 15 06:26:00 2022] __x64_sys_ioctl+0x16/0x20 [Wed Jun 15 06:26:00 2022] do_syscall_64+0x5b/0x1a0 [Wed Jun 15 06:26:00 2022] entry_SYSCALL_64_after_hwframe+0x65/0xca [Wed Jun 15 06:26:00 2022] RIP: 0033:0x7f600fb9563b [Wed Jun 15 06:26:00 2022] Code: Unable to access opcode bytes at RIP 0x7f600fb95611. [Wed Jun 15 06:26:00 2022] RSP: 002b:00007f600d9f4508 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 [Wed Jun 15 06:26:00 2022] RAX: ffffffffffffffda RBX: 0000557741094a80 RCX: 00007f600fb9563b [Wed Jun 15 06:26:00 2022] RDX: 00007f600d9f4530 RSI: 0000000000005a04 RDI: 000000000000000a [Wed Jun 15 06:26:00 2022] RBP: 00007f600d9f7b20 R08: 00007f6008065020 R09: 0000000000000003 [Wed Jun 15 06:26:00 2022] R10: 0000000000000090 R11: 0000000000000246 R12: 00007f600d9f4530 [Wed Jun 15 06:26:00 2022] R13: 7ca3107f4bd13204 R14: 000055773f1a7300 R15: 00007f600d9f7b90 [Wed Jun 15 06:26:00 2022] INFO: task zpool:221686 blocked for more than 120 seconds. [Wed Jun 15 06:26:00 2022] Tainted: P OE ---------r- - 4.18.0-305.10.2.x6.2.000.51.x86_64 #1 [Wed Jun 15 06:26:00 2022] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed Jun 15 06:26:00 2022] zpool D 0 221686 2243 0x00000080 [Wed Jun 15 06:26:00 2022] Call Trace: [Wed Jun 15 06:26:00 2022] __schedule+0x2c4/0x700 [Wed Jun 15 06:26:00 2022] schedule+0x38/0xa0 [Wed Jun 15 06:26:00 2022] schedule_preempt_disabled+0xa/0x10 [Wed Jun 15 06:26:00 2022] __mutex_lock.isra.6+0x2d0/0x4a0 [Wed Jun 15 06:26:00 2022] ? nvlist_xalloc.part.3+0x4b/0xb0 [zfs] [Wed Jun 15 06:26:00 2022] spa_all_configs+0x58/0x1f0 [zfs] [Wed Jun 15 06:26:00 2022] zfs_ioc_pool_configs+0x16/0x60 [zfs] [Wed Jun 15 06:26:00 2022] zfsdev_ioctl_common+0x656/0x980 [zfs] [Wed Jun 15 06:26:00 2022] zfsdev_ioctl+0x4f/0xe0 [zfs] [Wed Jun 15 06:26:00 2022] do_vfs_ioctl+0xa4/0x680 [Wed Jun 15 06:26:00 2022] ? hrtimer_nanosleep+0x89/0x110 [Wed Jun 15 06:26:00 2022] ? syscall_trace_enter+0x1d3/0x2c0 [Wed Jun 15 06:26:00 2022] ksys_ioctl+0x60/0x90 [Wed Jun 15 06:26:00 2022] __x64_sys_ioctl+0x16/0x20 [Wed Jun 15 06:26:00 2022] do_syscall_64+0x5b/0x1a0 [Wed Jun 15 06:26:00 2022] entry_SYSCALL_64_after_hwframe+0x65/0xca [Wed Jun 15 06:26:00 2022] RIP: 0033:0x7fd607ab463b [Wed Jun 15 06:26:00 2022] Code: Unable to access opcode bytes at RIP 0x7fd607ab4611. [Wed Jun 15 06:26:00 2022] RSP: 002b:00007fff23274428 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 [Wed Jun 15 06:26:00 2022] RAX: ffffffffffffffda RBX: 000055fb93715ac0 RCX: 00007fd607ab463b [Wed Jun 15 06:26:00 2022] RDX: 00007fff23274450 RSI: 0000000000005a04 RDI: 0000000000000003 [Wed Jun 15 06:26:00 2022] RBP: 00007fff23277a40 R08: 000055fb937a17e0 R09: 00007fd607d80c20 [Wed Jun 15 06:26:00 2022] R10: 000000000000004e R11: 0000000000000246 R12: 00007fff23274450 [Wed Jun 15 06:26:00 2022] R13: 0000000000000000 R14: 000055fb92131b30 R15: 000055fb93721630 [Wed Jun 15 06:26:00 2022] INFO: task zpool:261763 blocked for more than 120 seconds. [Wed Jun 15 06:26:00 2022] Tainted: P OE ---------r- - 4.18.0-305.10.2.x6.2.000.51.x86_64 #1 [Wed Jun 15 06:26:00 2022] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed Jun 15 06:26:00 2022] zpool D 0 261763 261761 0x00000080 [Wed Jun 15 06:26:00 2022] Call Trace: [Wed Jun 15 06:26:00 2022] __schedule+0x2c4/0x700 [Wed Jun 15 06:26:00 2022] schedule+0x38/0xa0 [Wed Jun 15 06:26:00 2022] schedule_preempt_disabled+0xa/0x10 [Wed Jun 15 06:26:00 2022] __mutex_lock.isra.6+0x2d0/0x4a0 [Wed Jun 15 06:26:00 2022] vdev_rebuild_get_stats+0xb5/0x260 [zfs] [Wed Jun 15 06:26:00 2022] vdev_config_generate+0x900/0xa70 [zfs] [Wed Jun 15 06:26:00 2022] ? __kmalloc_node+0x10b/0x290 [Wed Jun 15 06:26:00 2022] ? spl_kmem_alloc+0x118/0x160 [spl] [Wed Jun 15 06:26:00 2022] vdev_config_generate+0x32d/0xa70 [zfs] [Wed Jun 15 06:26:00 2022] ? nvlist_add_common+0x28c/0x3a0 [zfs] [Wed Jun 15 06:26:00 2022] ? kfree+0xc3/0x450 [Wed Jun 15 06:26:00 2022] ? spa_config_generate+0x1f1/0x4e0 [zfs] [Wed Jun 15 06:26:00 2022] spa_config_generate+0x227/0x4e0 [zfs] [Wed Jun 15 06:26:00 2022] spa_open_common+0x1ea/0x470 [zfs] [Wed Jun 15 06:26:00 2022] spa_get_stats+0x4d/0x4d0 [zfs] [Wed Jun 15 06:26:00 2022] ? _cond_resched+0x15/0x30 [Wed Jun 15 06:26:00 2022] ? __kmalloc_node+0x10b/0x290 [Wed Jun 15 06:26:00 2022] ? spl_kmem_alloc+0x118/0x160 [spl] [Wed Jun 15 06:26:00 2022] zfs_ioc_pool_stats+0x34/0x80 [zfs] [Wed Jun 15 06:26:00 2022] zfsdev_ioctl_common+0x656/0x980 [zfs] [Wed Jun 15 06:26:00 2022] zfsdev_ioctl+0x4f/0xe0 [zfs] [Wed Jun 15 06:26:00 2022] do_vfs_ioctl+0xa4/0x680 [Wed Jun 15 06:26:00 2022] ? handle_mm_fault+0xc2/0x1d0 [Wed Jun 15 06:26:00 2022] ? syscall_trace_enter+0x1d3/0x2c0 [Wed Jun 15 06:26:00 2022] ksys_ioctl+0x60/0x90 [Wed Jun 15 06:26:00 2022] __x64_sys_ioctl+0x16/0x20 [Wed Jun 15 06:26:00 2022] do_syscall_64+0x5b/0x1a0 [Wed Jun 15 06:26:00 2022] entry_SYSCALL_64_after_hwframe+0x65/0xca [Wed Jun 15 06:26:00 2022] RIP: 0033:0x7faea054c63b [Wed Jun 15 06:26:00 2022] Code: Unable to access opcode bytes at RIP 0x7faea054c611. [Wed Jun 15 06:26:00 2022] RSP: 002b:00007fff7a48a058 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 [Wed Jun 15 06:26:00 2022] RAX: ffffffffffffffda RBX: 00007fff7a48a080 RCX: 00007faea054c63b [Wed Jun 15 06:26:00 2022] RDX: 00007fff7a48a080 RSI: 0000000000005a05 RDI: 0000000000000004 [Wed Jun 15 06:26:00 2022] RBP: 00007fff7a48d670 R08: 00005630db89f580 R09: 0000000000000003 [Wed Jun 15 06:26:00 2022] R10: 0000000000000014 R11: 0000000000000246 R12: 00005630db89f430 [Wed Jun 15 06:26:00 2022] R13: 00005630db892940 R14: 00007fff7a48d694 R15: 00005630db89dbc0 [Wed Jun 15 06:26:00 2022] INFO: task vdev_rebuild:261796 blocked for more than 120 seconds. [Wed Jun 15 06:26:00 2022] Tainted: P OE ---------r- - 4.18.0-305.10.2.x6.2.000.51.x86_64 #1 [Wed Jun 15 06:26:00 2022] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed Jun 15 06:26:00 2022] vdev_rebuild D 0 261796 2 0x80004080 [Wed Jun 15 06:26:00 2022] Call Trace: [Wed Jun 15 06:26:00 2022] __schedule+0x2c4/0x700 [Wed Jun 15 06:26:00 2022] schedule+0x38/0xa0 [Wed Jun 15 06:26:00 2022] spl_panic+0xf9/0xfb [spl] [Wed Jun 15 06:26:00 2022] ? kvm_sched_clock_read+0xd/0x20 [Wed Jun 15 06:26:00 2022] ? __switch_to_asm+0x41/0x70 [Wed Jun 15 06:26:00 2022] ? __switch_to_asm+0x35/0x70 [Wed Jun 15 06:26:00 2022] ? __switch_to_asm+0x41/0x70 [Wed Jun 15 06:26:00 2022] ? __switch_to_asm+0x35/0x70 [Wed Jun 15 06:26:00 2022] ? __switch_to_asm+0x41/0x70 [Wed Jun 15 06:26:00 2022] ? __switch_to_asm+0x35/0x70 [Wed Jun 15 06:26:00 2022] ? __switch_to_asm+0x41/0x70 [Wed Jun 15 06:26:00 2022] ? _cond_resched+0x15/0x30 [Wed Jun 15 06:26:00 2022] ? mutex_lock+0xe/0x30 [Wed Jun 15 06:26:00 2022] ? vdev_rebuild_range+0xa50/0xa50 [zfs] [Wed Jun 15 06:26:00 2022] ? spl_taskq_fini+0x70/0x70 [spl] [Wed Jun 15 06:26:00 2022] vdev_rebuild_thread+0xbb6/0xdb0 [zfs] [Wed Jun 15 06:26:00 2022] ? thread_generic_wrapper+0x6b/0xb0 [spl] [Wed Jun 15 06:26:00 2022] ? vdev_rebuild_range+0xa50/0xa50 [zfs] [Wed Jun 15 06:26:00 2022] ? spl_taskq_fini+0x70/0x70 [spl] [Wed Jun 15 06:26:00 2022] thread_generic_wrapper+0x78/0xb0 [spl] [Wed Jun 15 06:26:00 2022] kthread+0x116/0x130 [Wed Jun 15 06:26:00 2022] ? kthread_flush_work_fn+0x10/0x10 [Wed Jun 15 06:26:00 2022] ret_from_fork+0x35/0x40 [Wed Jun 15 06:26:00 2022] INFO: task zpool:261924 blocked for more than 120 seconds. [Wed Jun 15 06:26:00 2022] Tainted: P OE ---------r- - 4.18.0-305.10.2.x6.2.000.51.x86_64 #1 [Wed Jun 15 06:26:00 2022] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Wed Jun 15 06:26:00 2022] zpool D 0 261924 261918 0x00004084 [Wed Jun 15 06:26:00 2022] Call Trace: [Wed Jun 15 06:26:00 2022] __schedule+0x2c4/0x700 [Wed Jun 15 06:26:00 2022] schedule+0x38/0xa0 [Wed Jun 15 06:26:00 2022] schedule_preempt_disabled+0xa/0x10 [Wed Jun 15 06:26:00 2022] __mutex_lock.isra.6+0x2d0/0x4a0 [Wed Jun 15 06:26:00 2022] ? nvlist_xalloc.part.3+0x4b/0xb0 [zfs] [Wed Jun 15 06:26:00 2022] spa_all_configs+0x58/0x1f0 [zfs] [Wed Jun 15 06:26:00 2022] zfs_ioc_pool_configs+0x16/0x60 [zfs] [Wed Jun 15 06:26:00 2022] zfsdev_ioctl_common+0x656/0x980 [zfs] [Wed Jun 15 06:26:00 2022] zfsdev_ioctl+0x4f/0xe0 [zfs] [Wed Jun 15 06:26:00 2022] do_vfs_ioctl+0xa4/0x680 [Wed Jun 15 06:26:00 2022] ? handle_mm_fault+0xc2/0x1d0 [Wed Jun 15 06:26:00 2022] ? syscall_trace_enter+0x1d3/0x2c0 [Wed Jun 15 06:26:00 2022] ksys_ioctl+0x60/0x90 [Wed Jun 15 06:26:00 2022] __x64_sys_ioctl+0x16/0x20 [Wed Jun 15 06:26:00 2022] do_syscall_64+0x5b/0x1a0 [Wed Jun 15 06:26:00 2022] entry_SYSCALL_64_after_hwframe+0x65/0xca [Wed Jun 15 06:26:00 2022] RIP: 0033:0x7f7f0098463b [Wed Jun 15 06:26:00 2022] Code: Unable to access opcode bytes at RIP 0x7f7f00984611. [Wed Jun 15 06:26:00 2022] RSP: 002b:00007ffeaeb6ecf8 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 [Wed Jun 15 06:26:00 2022] RAX: ffffffffffffffda RBX: 000055c0d7826ac0 RCX: 00007f7f0098463b [Wed Jun 15 06:26:00 2022] RDX: 00007ffeaeb6ed20 RSI: 0000000000005a04 RDI: 0000000000000003 [Wed Jun 15 06:26:00 2022] RBP: 00007ffeaeb72310 R08: 00007f7f03059010 R09: 0000000000000000 [Wed Jun 15 06:26:00 2022] R10: 0000000000000022 R11: 0000000000000246 R12: 00007ffeaeb6ed20 [Wed Jun 15 06:26:00 2022] R13: 0000000000000000 R14: 000055c0d6486b30 R15: 000055c0d7826620 ==END OF CASE3=============================