CASE3: ===== DRAID2: ======= + truncate -s 1G file1 file2 file3 file4 file5 file6 file7 file8 file9 file10 file11 file12 file13 file14 file15 file16 file17 file18 file19 file20 + pwd /root/test/test/files + zpool create -f -o cachefile=none -o failmode=panic -o multihost=on -O canmount=off pool-oss0 draid2:16d:2s /root/test/test/files/file1 /root/test/test/files/file2 /root/test/test/files/file3 /root/test/test/files/file4 /root/test/test/files/file5 /root/test/test/files/file6 /root/test/test/files/file7 /root/test/test/files/file8 /root/test/test/files/file9 /root/test/test/files/file10 /root/test/test/files/file11 /root/test/test/files/file12 /root/test/test/files/file13 /root/test/test/files/file14 /root/test/test/files/file15 /root/test/test/files/file16 /root/test/test/files/file17 /root/test/test/files/file18 /root/test/test/files/file19 /root/test/test/files/file20 + zfs create -o mountpoint=/mnt/ost0 -o compression=off pool-oss0/ost0 + zfs set recordsize=4M pool-oss0 [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# [root@localhost files]# fio --name=pat_test --ioengine=libaio --fallocate=none --iodepth=4 --rw=write --bs=1M --size=2G --numjobs=6 --do_verify=0 --verify=pattern --verify_pattern=0x0a0b0c0d01020304 --directory=/mnt/ost0/ --group_reporting pat_test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes pat_test: Laying out IO file (1 file / 2048MiB) pat_test: Laying out IO file (1 file / 2048MiB) pat_test: Laying out IO file (1 file / 2048MiB) pat_test: Laying out IO file (1 file / 2048MiB) pat_test: Laying out IO file (1 file / 2048MiB) pat_test: Laying out IO file (1 file / 2048MiB) Jobs: 1 (f=0): [_(3),f(1),_(2)][100.0%][w=137MiB/s][w=136 IOPS][eta 00m:00s] pat_test: (groupid=0, jobs=6): err= 0: pid=304838: Fri Jun 17 04:55:13 2022 write: IOPS=257, BW=258MiB/s (270MB/s)(12.0GiB/47716msec); 0 zone resets slat (usec): min=117, max=1416.2k, avg=20693.98, stdev=99767.28 clat (usec): min=2, max=1607.6k, avg=64761.61, stdev=173226.32 lat (usec): min=187, max=2133.6k, avg=85467.35, stdev=198468.77 clat percentiles (usec): | 1.00th=[ 1663], 5.00th=[ 3523], 10.00th=[ 5211], | 20.00th=[ 7570], 30.00th=[ 8979], 40.00th=[ 10290], | 50.00th=[ 11600], 60.00th=[ 12911], 70.00th=[ 14484], | 80.00th=[ 16909], 90.00th=[ 85459], 95.00th=[ 549454], | 99.00th=[ 725615], 99.50th=[ 901776], 99.90th=[1400898], | 99.95th=[1417675], 99.99th=[1602225] bw ( KiB/s): min=16286, max=1290124, per=100.00%, avg=325282.15, stdev=40430.58, samples=452 iops : min= 10, max= 1257, avg=313.72, stdev=39.54, samples=452 lat (usec) : 4=0.01%, 10=0.01%, 20=0.02%, 50=0.02%, 500=0.01% lat (usec) : 1000=0.02% lat (msec) : 2=1.94%, 4=4.19%, 10=31.61%, 20=46.39%, 50=5.13% lat (msec) : 100=0.88%, 250=0.75%, 500=1.64%, 750=6.53%, 1000=0.56% lat (msec) : 2000=0.30% cpu : usr=5.22%, sys=9.13%, ctx=3877, majf=0, minf=92 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=0,12288,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): WRITE: bw=258MiB/s (270MB/s), 258MiB/s-258MiB/s (270MB/s-270MB/s), io=12.0GiB (12.9GB), run=47716-47716msec [root@localhost files]# [root@localhost files]# fio --name=pat_test --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --do_verify=1 --verify=pattern --verify_pattern=0x0a0b0c0d01020304 --directory=/mnt/ost0/ --group_reporting pat_test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 3 (f=3): [_(1),V(1),_(1),V(1),_(1),V(1)][90.9%][r=620MiB/s][r=620 IOPS][eta 00m:02s] pat_test: (groupid=0, jobs=6): err= 0: pid=318795: Fri Jun 17 04:56:59 2022 read: IOPS=625, BW=625MiB/s (656MB/s)(12.0GiB/19646msec) slat (usec): min=94, max=250567, avg=8790.47, stdev=18901.31 clat (usec): min=70, max=251413, avg=27559.24, stdev=25965.42 lat (usec): min=188, max=251813, avg=36354.12, stdev=25210.77 clat percentiles (usec): | 1.00th=[ 1188], 5.00th=[ 1745], 10.00th=[ 2442], 20.00th=[ 4113], | 30.00th=[ 10290], 40.00th=[ 15139], 50.00th=[ 20055], 60.00th=[ 27132], | 70.00th=[ 35390], 80.00th=[ 45876], 90.00th=[ 62129], 95.00th=[ 78119], | 99.00th=[112722], 99.50th=[131597], 99.90th=[166724], 99.95th=[187696], | 99.99th=[250610] bw ( KiB/s): min=345273, max=1189186, per=100.00%, avg=658402.85, stdev=29506.01, samples=219 iops : min= 333, max= 1159, avg=639.53, stdev=28.89, samples=219 lat (usec) : 100=0.01%, 250=0.02%, 500=0.02%, 750=0.05%, 1000=0.21% lat (msec) : 2=6.68%, 4=12.61%, 10=9.90%, 20=20.22%, 50=33.37% lat (msec) : 100=15.09%, 250=1.79%, 500=0.02% cpu : usr=3.60%, sys=6.82%, ctx=6108, majf=0, minf=5708 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=625MiB/s (656MB/s), 625MiB/s-625MiB/s (656MB/s-656MB/s), io=12.0GiB (12.9GB), run=19646-19646msec [root@localhost files]# md5sum -c md5sum /mnt/ost0/pat_test.0.0: OK /mnt/ost0/pat_test.1.0: OK /mnt/ost0/pat_test.2.0: OK /mnt/ost0/pat_test.3.0: OK /mnt/ost0/pat_test.4.0: OK /mnt/ost0/pat_test.5.0: OK [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# dd if=/dev/zero of=file1 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.540298 s, 2.0 GB/s [root@localhost files]# dd if=/dev/zero of=file10 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.634796 s, 1.7 GB/s [root@localhost files]# [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# [root@localhost files]# fio --name=pat_test --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --do_verify=1 --verify=pattern --verify_pattern=0x0a0b0c0d01020304 --directory=/mnt/ost0/ --group_reporting pat_test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 1 (f=1): [_(4),V(1),_(1)][100.0%][r=27.3MiB/s][r=27 IOPS][eta 00m:00s] pat_test: (groupid=0, jobs=6): err= 0: pid=322257: Fri Jun 17 05:00:48 2022 read: IOPS=77, BW=77.9MiB/s (81.7MB/s)(12.0GiB/157667msec) slat (usec): min=112, max=1048.7k, avg=72440.62, stdev=151510.60 clat (usec): min=205, max=1050.9k, avg=219208.67, stdev=197318.08 lat (usec): min=620, max=1051.3k, avg=291658.87, stdev=178548.84 clat percentiles (usec): | 1.00th=[ 1811], 5.00th=[ 2573], 10.00th=[ 3163], | 20.00th=[ 5669], 30.00th=[ 34341], 40.00th=[ 105382], | 50.00th=[ 206570], 60.00th=[ 278922], 70.00th=[ 333448], | 80.00th=[ 400557], 90.00th=[ 492831], 95.00th=[ 557843], | 99.00th=[ 717226], 99.50th=[ 784335], 99.90th=[ 926942], | 99.95th=[ 985662], 99.99th=[1052771] bw ( KiB/s): min=15359, max=584952, per=100.00%, avg=86875.64, stdev=12261.18, samples=1669 iops : min= 10, max= 568, avg=80.50, stdev=12.03, samples=1669 lat (usec) : 250=0.02%, 500=0.02%, 750=0.01%, 1000=0.02% lat (msec) : 2=1.62%, 4=13.69%, 10=7.73%, 20=3.12%, 50=6.69% lat (msec) : 100=6.50%, 250=16.20%, 500=34.89%, 750=8.89%, 1000=0.58% lat (msec) : 2000=0.02% cpu : usr=0.55%, sys=1.15%, ctx=5892, majf=1, minf=6241 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=77.9MiB/s (81.7MB/s), 77.9MiB/s-77.9MiB/s (81.7MB/s-81.7MB/s), io=12.0GiB (12.9GB), run=157667-157667msec [root@localhost files]# [root@localhost files]# md5sum -c md5sum /mnt/ost0/pat_test.0.0: OK /mnt/ost0/pat_test.1.0: OK /mnt/ost0/pat_test.2.0: OK /mnt/ost0/pat_test.3.0: OK /mnt/ost0/pat_test.4.0: OK /mnt/ost0/pat_test.5.0: OK [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an unrecoverable error. An attempt was made to correct the error. Applications are unaffected. action: Determine if the device needs to be replaced, and clear the errors using 'zpool clear' or replace the device with 'zpool replace'. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-9P scan: scrub repaired 1.55G in 00:01:52 with 3 errors on Fri Jun 17 05:00:50 2022 scan: resilvered (draid2:16d:20c:2s-0) 688M in 00:00:43 with 0 errors on Fri Jun 17 04:58:58 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 draid2:16d:20c:2s-0 DEGRADED 0 0 0 spare-0 DEGRADED 0 0 2.81K /root/test/test/files/file1 DEGRADED 0 0 66 too many errors draid2-0-0 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 8 /root/test/test/files/file3 ONLINE 0 0 8 /root/test/test/files/file4 ONLINE 0 0 12 /root/test/test/files/file5 ONLINE 0 0 12 /root/test/test/files/file6 ONLINE 0 0 8 /root/test/test/files/file7 ONLINE 0 0 12 /root/test/test/files/file8 ONLINE 0 0 12 /root/test/test/files/file9 ONLINE 0 0 12 spare-9 DEGRADED 0 0 3.33K /root/test/test/files/file10 DEGRADED 0 0 694 too many errors draid2-0-1 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 12 /root/test/test/files/file12 ONLINE 0 0 12 /root/test/test/files/file13 ONLINE 0 0 8 /root/test/test/files/file14 ONLINE 0 0 12 /root/test/test/files/file15 ONLINE 0 0 12 /root/test/test/files/file16 ONLINE 0 0 12 /root/test/test/files/file17 ONLINE 0 0 12 /root/test/test/files/file18 ONLINE 0 0 12 /root/test/test/files/file19 ONLINE 0 0 8 /root/test/test/files/file20 ONLINE 0 0 8 spares draid2-0-0 INUSE currently in use draid2-0-1 INUSE currently in use errors: No known data errors [root@localhost files]# dd if=/dev/zero of=file3 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.570282 s, 1.9 GB/s [root@localhost files]# dd if=/dev/zero of=file13 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.625452 s, 1.7 GB/s [root@localhost files]# [root@localhost files]# fio --name=pat_test --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --do_verify=1 --verify=pattern --verify_pattern=0x0a0b0c0d01020304 --directory=/mnt/ost0/ --group_reporting pat_test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes fio: io_u error on file /mnt/ost0//pat_test.0.0: Input/output error: read offset=226492416, buflen=1048576 fio: pid=356125, err=5/file:io_u.c:1803, func=io_u error, error=Input/output error fio: io_u error on file /mnt/ost0//pat_test.3.0: Input/output error: read offset=549453824, buflen=1048576 fio: pid=356128, err=5/file:io_u.c:1803, func=io_u error, error=Input/output error fio: io_u error on file /mnt/ost0//pat_test.5.0: Input/output error: read offset=683671552, buflen=1048576 fio: pid=356126, err=5/file:io_u.c:1803, func=io_u error, error=Input/output error fio: io_u error on file /mnt/ost0//pat_test.2.0: Input/output error: read offset=1069547520, buflen=1048576 fio: pid=356129, err=5/file:io_u.c:1803, func=io_u error, error=Input/output error fio: io_u error on file /mnt/ost0//pat_test.4.0: Input/output error: read offset=1778384896, buflen=1048576 fio: pid=356127, err=5/file:io_u.c:1803, func=io_u error, error=Input/output error fio: io_u error on file /mnt/ost0//pat_test.1.0: Input/output error: read offset=1837105152, buflen=1048576 fio: pid=356130, err=5/file:io_u.c:1803, func=io_u error, error=Input/output error pat_test: (groupid=0, jobs=6): err= 5 (file:io_u.c:1803, func=io_u error, error=Input/output error): pid=356125: Fri Jun 17 05:04:19 2022 read: IOPS=59, BW=59.6MiB/s (62.5MB/s)(5860MiB/98250msec) slat (usec): min=111, max=2297.3k, avg=62436.19, stdev=140413.38 clat (usec): min=831, max=5157.2k, avg=182437.44, stdev=204796.03 lat (msec): min=2, max=5158, avg=240.94, stdev=191.36 clat percentiles (usec): | 1.00th=[ 1319], 5.00th=[ 1811], 10.00th=[ 2245], | 20.00th=[ 3458], 30.00th=[ 40109], 40.00th=[ 112722], | 50.00th=[ 154141], 60.00th=[ 193987], 70.00th=[ 254804], | 80.00th=[ 329253], 90.00th=[ 404751], 95.00th=[ 471860], | 99.00th=[ 650118], 99.50th=[ 700449], 99.90th=[2566915], | 99.95th=[3506439], 99.99th=[5133829] bw ( KiB/s): min=10679, max=435844, per=100.00%, avg=94645.62, stdev=13756.88, samples=671 iops : min= 6, max= 423, avg=88.61, stdev=13.51, samples=671 lat (usec) : 1000=0.05% lat (msec) : 2=7.27%, 4=14.56%, 10=2.86%, 20=1.21%, 50=6.44% lat (msec) : 100=6.27%, 250=30.34%, 500=26.72%, 750=3.54%, 1000=0.08% lat (msec) : 2000=0.08%, >=2000=0.17% cpu : usr=0.49%, sys=1.09%, ctx=2209, majf=1, minf=6299 IO depths : 1=0.1%, 2=0.2%, 4=99.7%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=5884,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=59.6MiB/s (62.5MB/s), 59.6MiB/s-59.6MiB/s (62.5MB/s-62.5MB/s), io=5860MiB (6145MB), run=98250-98250msec [root@localhost files]# [root@localhost files]# md5sum -c md5sum md5sum: /mnt/ost0/pat_test.0.0: Input/output error /mnt/ost0/pat_test.0.0: FAILED open or read md5sum: /mnt/ost0/pat_test.1.0: Input/output error /mnt/ost0/pat_test.1.0: FAILED open or read md5sum: /mnt/ost0/pat_test.2.0: Input/output error /mnt/ost0/pat_test.2.0: FAILED open or read md5sum: /mnt/ost0/pat_test.3.0: Input/output error /mnt/ost0/pat_test.3.0: FAILED open or read md5sum: /mnt/ost0/pat_test.4.0: Input/output error /mnt/ost0/pat_test.4.0: FAILED open or read md5sum: /mnt/ost0/pat_test.5.0: Input/output error /mnt/ost0/pat_test.5.0: FAILED open or read md5sum: WARNING: 6 listed files could not be read [root@localhost files]#zpool status -v 2 pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an error resulting in data corruption. Applications may be affected. action: Restore the file in question if possible. Otherwise restore the entire pool from backup. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-8A scan: scrub repaired 1.55G in 00:01:52 with 3 errors on Fri Jun 17 05:00:50 2022 scan: resilvered (draid2:16d:20c:2s-0) 688M in 00:00:43 with 0 errors on Fri Jun 17 04:58:58 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 draid2:16d:20c:2s-0 DEGRADED 0 0 0 spare-0 DEGRADED 0 0 2.99K /root/test/test/files/file1 DEGRADED 0 0 66 too many errors draid2-0-0 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 56 /root/test/test/files/file3 DEGRADED 0 0 1.28K too many errors /root/test/test/files/file4 ONLINE 0 0 60 /root/test/test/files/file5 ONLINE 0 0 68 /root/test/test/files/file6 ONLINE 0 0 72 /root/test/test/files/file7 ONLINE 0 0 76 /root/test/test/files/file8 ONLINE 0 0 76 /root/test/test/files/file9 ONLINE 0 0 76 spare-9 DEGRADED 0 0 3.42K /root/test/test/files/file10 DEGRADED 0 0 694 too many errors draid2-0-1 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 68 /root/test/test/files/file12 ONLINE 0 0 52 /root/test/test/files/file13 DEGRADED 0 0 1.24K too many errors /root/test/test/files/file14 ONLINE 0 0 68 /root/test/test/files/file15 ONLINE 0 0 76 /root/test/test/files/file16 ONLINE 0 0 76 /root/test/test/files/file17 ONLINE 0 0 76 /root/test/test/files/file18 ONLINE 0 0 76 /root/test/test/files/file19 ONLINE 0 0 72 /root/test/test/files/file20 ONLINE 0 0 72 spares draid2-0-0 INUSE currently in use draid2-0-1 INUSE currently in use errors: Permanent errors have been detected in the following files: /mnt/ost0/pat_test.0.0 /mnt/ost0/pat_test.5.0 /mnt/ost0/pat_test.3.0 /mnt/ost0/pat_test.2.0 pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an error resulting in data corruption. Applications may be affected. action: Restore the file in question if possible. Otherwise restore the entire pool from backup. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-8A scan: scrub repaired 1.55G in 00:01:52 with 3 errors on Fri Jun 17 05:00:50 2022 scan: resilvered (draid2:16d:20c:2s-0) 688M in 00:00:43 with 0 errors on Fri Jun 17 04:58:58 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 draid2:16d:20c:2s-0 DEGRADED 0 0 0 spare-0 DEGRADED 0 0 3.00K /root/test/test/files/file1 DEGRADED 0 0 66 too many errors draid2-0-0 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 66 /root/test/test/files/file3 DEGRADED 0 0 1.29K too many errors /root/test/test/files/file4 ONLINE 0 0 65 /root/test/test/files/file5 ONLINE 0 0 78 /root/test/test/files/file6 DEGRADED 0 0 82 too many errors /root/test/test/files/file7 ONLINE 0 0 86 /root/test/test/files/file8 ONLINE 0 0 86 /root/test/test/files/file9 ONLINE 0 0 86 spare-9 DEGRADED 0 0 3.43K /root/test/test/files/file10 DEGRADED 0 0 694 too many errors draid2-0-1 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 73 /root/test/test/files/file12 ONLINE 0 0 62 /root/test/test/files/file13 DEGRADED 0 0 1.26K too many errors /root/test/test/files/file14 ONLINE 0 0 78 /root/test/test/files/file15 ONLINE 0 0 86 /root/test/test/files/file16 ONLINE 0 0 86 /root/test/test/files/file17 ONLINE 0 0 86 /root/test/test/files/file18 ONLINE 0 0 86 /root/test/test/files/file19 ONLINE 0 0 82 /root/test/test/files/file20 ONLINE 0 0 82 spares draid2-0-0 INUSE currently in use draid2-0-1 INUSE currently in use munmap_chunk(): invalid pointer Aborted (core dumped) [root@localhost files]# zpool status -v pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an error resulting in data corruption. Applications may be affected. action: Restore the file in question if possible. Otherwise restore the entire pool from backup. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-8A scan: scrub repaired 1.55G in 00:01:52 with 3 errors on Fri Jun 17 05:00:50 2022 scan: resilvered (draid2:16d:20c:2s-0) 688M in 00:00:43 with 0 errors on Fri Jun 17 04:58:58 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 draid2:16d:20c:2s-0 DEGRADED 0 0 0 spare-0 DEGRADED 0 0 3.07K /root/test/test/files/file1 DEGRADED 0 0 66 too many errors draid2-0-0 ONLINE 0 0 0 /root/test/test/files/file2 DEGRADED 0 0 98 too many errors /root/test/test/files/file3 DEGRADED 0 0 1.38K too many errors /root/test/test/files/file4 DEGRADED 0 0 102 too many errors /root/test/test/files/file5 DEGRADED 0 0 122 too many errors /root/test/test/files/file6 DEGRADED 0 0 128 too many errors /root/test/test/files/file7 DEGRADED 0 0 132 too many errors /root/test/test/files/file8 DEGRADED 0 0 132 too many errors /root/test/test/files/file9 DEGRADED 0 0 132 too many errors spare-9 DEGRADED 0 0 3.47K /root/test/test/files/file10 DEGRADED 0 0 694 too many errors draid2-0-1 ONLINE 0 0 0 /root/test/test/files/file11 DEGRADED 0 0 112 too many errors /root/test/test/files/file12 DEGRADED 0 0 102 too many errors /root/test/test/files/file13 DEGRADED 0 0 1.37K too many errors /root/test/test/files/file14 DEGRADED 0 0 122 too many errors /root/test/test/files/file15 DEGRADED 0 0 132 too many errors /root/test/test/files/file16 DEGRADED 0 0 132 too many errors /root/test/test/files/file17 DEGRADED 0 0 132 too many errors /root/test/test/files/file18 DEGRADED 0 0 132 too many errors /root/test/test/files/file19 DEGRADED 0 0 118 too many errors /root/test/test/files/file20 DEGRADED 0 0 118 too many errors spares draid2-0-0 INUSE currently in use draid2-0-1 INUSE currently in use errors: Permanent errors have been detected in the following files: /mnt/ost0/pat_test.0.0 /mnt/ost0/pat_test.5.0 /mnt/ost0/pat_test.4.0 /mnt/ost0/pat_test.3.0 /mnt/ost0/pat_test.2.0 /mnt/ost0/pat_test.1.0 [root@localhost files]# [root@localhost files]# zpool clear pool-oss0 [root@localhost files]# zpool scrub pool-oss0 [root@localhost files]# md5sum -c md5sum /mnt/ost0/pat_test.0.0: OK /mnt/ost0/pat_test.1.0: OK /mnt/ost0/pat_test.2.0: OK /mnt/ost0/pat_test.3.0: OK /mnt/ost0/pat_test.4.0: OK /mnt/ost0/pat_test.5.0: OK pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an error resulting in data corruption. Applications may be affected. action: Restore the file in question if possible. Otherwise restore the entire pool from backup. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-8A scan: scrub repaired 726M in 00:01:35 with 0 errors on Fri Jun 17 05:00:50 2022 scan: resilvered (draid2:16d:20c:2s-0) 688M in 00:00:43 with 0 errors on Fri Jun 17 04:58:58 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 1.46K /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 1.48K /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors [root@localhost files]# zpool clear pool-oss0 [root@localhost files]# sync; echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# md5sum -c md5sum /mnt/ost0/pat_test.0.0: OK /mnt/ost0/pat_test.1.0: OK /mnt/ost0/pat_test.2.0: OK /mnt/ost0/pat_test.3.0: OK /mnt/ost0/pat_test.4.0: OK /mnt/ost0/pat_test.5.0: OK [root@localhost files]# pool: pool-oss0 state: ONLINE scan: scrub repaired 726M in 00:01:35 with 0 errors on Fri Jun 17 06:18:31 2022 scan: resilvered (draid2:16d:20c:2s-0) 678M in 00:00:35 with 0 errors on Fri Jun 17 06:13:06 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 draid2:16d:20c:2s-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file20 ONLINE 0 0 0 spares draid2-0-0 AVAIL draid2-0-1 AVAIL errors: No known data errors RAIDZ2: ====== + truncate -s 1G file1 file2 file3 file4 file5 file6 file7 file8 file9 file10 file11 file12 file13 file14 file15 file16 file17 file18 file19 file20 + pwd /root/test/test/files + zpool create -f -o cachefile=none -o failmode=panic -O canmount=off pool-oss0 raidz2 /root/test/test/files/file1 /root/test/test/files/file2 /root/test/test/files/file3 /root/test/test/files/file4 /root/test/test/files/file5 /root/test/test/files/file6 /root/test/test/files/file7 /root/test/test/files/file8 /root/test/test/files/file9 /root/test/test/files/file10 /root/test/test/files/file11 /root/test/test/files/file12 /root/test/test/files/file13 /root/test/test/files/file14 /root/test/test/files/file15 /root/test/test/files/file16 /root/test/test/files/file17 /root/test/test/files/file18 spare /root/test/test/files/file19 /root/test/test/files/file20 + zfs create -o mountpoint=/mnt/ost0 -o compression=off pool-oss0/ost0 + zfs set recordsize=4M pool-oss0 [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 raidz2-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 spares /root/test/test/files/file19 AVAIL /root/test/test/files/file20 AVAIL errors: No known data errors [root@localhost files]# fio --name=pat_test --ioengine=libaio --fallocate=none --iodepth=4 --rw=write --bs=1M --size=2G --numjobs=6 --do_verify=0 --verify=pattern --verify_pattern=0x0a0b0c0d01020304 --directory=/mnt/ost0/ --group_reporting pat_test: (g=0): rw=write, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes pat_test: Laying out IO file (1 file / 2048MiB) pat_test: Laying out IO file (1 file / 2048MiB) pat_test: Laying out IO file (1 file / 2048MiB) pat_test: Laying out IO file (1 file / 2048MiB) pat_test: Laying out IO file (1 file / 2048MiB) pat_test: Laying out IO file (1 file / 2048MiB) Jobs: 2 (f=1): [_(2),f(1),_(2),W(1)][95.3%][w=333MiB/s][w=333 IOPS][eta 00m:02s] pat_test: (groupid=0, jobs=6): err= 0: pid=407938: Fri Jun 17 05:16:18 2022 write: IOPS=298, BW=298MiB/s (313MB/s)(12.0GiB/41209msec); 0 zone resets slat (usec): min=117, max=1296.4k, avg=17087.43, stdev=86199.27 clat (usec): min=5, max=1307.1k, avg=53850.43, stdev=150212.64 lat (usec): min=157, max=1309.8k, avg=70946.73, stdev=172555.79 clat percentiles (usec): | 1.00th=[ 1074], 5.00th=[ 2147], 10.00th=[ 4490], | 20.00th=[ 7308], 30.00th=[ 8848], 40.00th=[ 10028], | 50.00th=[ 11338], 60.00th=[ 12649], 70.00th=[ 14091], | 80.00th=[ 15926], 90.00th=[ 44827], 95.00th=[ 434111], | 99.00th=[ 767558], 99.50th=[ 859833], 99.90th=[1300235], | 99.95th=[1300235], 99.99th=[1300235] bw ( KiB/s): min=27735, max=2333442, per=100.00%, avg=396696.13, stdev=64388.89, samples=387 iops : min= 22, max= 2276, avg=383.63, stdev=62.94, samples=387 lat (usec) : 10=0.03%, 50=0.02%, 500=0.02%, 750=0.24%, 1000=0.37% lat (msec) : 2=3.85%, 4=4.48%, 10=31.14%, 20=46.44%, 50=3.62% lat (msec) : 100=0.55%, 250=1.16%, 500=5.31%, 750=1.72%, 1000=0.72% lat (msec) : 2000=0.33% cpu : usr=6.56%, sys=10.72%, ctx=3486, majf=0, minf=95 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=0,12288,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): WRITE: bw=298MiB/s (313MB/s), 298MiB/s-298MiB/s (313MB/s-313MB/s), io=12.0GiB (12.9GB), run=41209-41209msec [root@localhost files]# [root@localhost files]# fio --name=pat_test --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --do_verify=1 --verify=pattern --verify_pattern=0x0a0b0c0d01020304 --directory=/mnt/ost0/ --group_reporting pat_test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 4 (f=4): [_(1),V(3),_(1),V(1)][91.3%][r=532MiB/s][r=532 IOPS][eta 00m:02s] pat_test: (groupid=0, jobs=6): err= 0: pid=413982: Fri Jun 17 05:16:49 2022 read: IOPS=587, BW=587MiB/s (616MB/s)(12.0GiB/20921msec) slat (usec): min=92, max=222280, avg=9472.43, stdev=19514.49 clat (usec): min=82, max=224309, avg=29693.71, stdev=26424.07 lat (usec): min=268, max=224953, avg=39171.87, stdev=25164.96 clat percentiles (usec): | 1.00th=[ 1254], 5.00th=[ 2024], 10.00th=[ 2802], 20.00th=[ 5080], | 30.00th=[ 11731], 40.00th=[ 17695], 50.00th=[ 23987], 60.00th=[ 30802], | 70.00th=[ 39060], 80.00th=[ 48497], 90.00th=[ 63177], 95.00th=[ 79168], | 99.00th=[115868], 99.50th=[135267], 99.90th=[179307], 99.95th=[193987], | 99.99th=[223347] bw ( KiB/s): min=354474, max=1021421, per=100.00%, avg=617717.84, stdev=23511.77, samples=238 iops : min= 342, max= 995, avg=599.96, stdev=23.01, samples=238 lat (usec) : 100=0.01%, 250=0.02%, 500=0.02%, 750=0.02%, 1000=0.08% lat (msec) : 2=4.60%, 4=11.84%, 10=11.06%, 20=16.02%, 50=37.46% lat (msec) : 100=16.69%, 250=2.18% cpu : usr=3.52%, sys=8.22%, ctx=6106, majf=0, minf=6224 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=587MiB/s (616MB/s), 587MiB/s-587MiB/s (616MB/s-616MB/s), io=12.0GiB (12.9GB), run=20921-20921msec [root@localhost files]# [root@localhost files]# md5sum -c md5sum /mnt/ost0/pat_test.0.0: OK /mnt/ost0/pat_test.1.0: OK /mnt/ost0/pat_test.2.0: OK /mnt/ost0/pat_test.3.0: OK /mnt/ost0/pat_test.4.0: OK /mnt/ost0/pat_test.5.0: OK [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 raidz2-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 spares /root/test/test/files/file19 AVAIL /root/test/test/files/file20 AVAIL errors: No known data errors [root@localhost files]# dd if=/dev/zero of=file1 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.991791 s, 1.1 GB/s [root@localhost files]# [root@localhost files]# dd if=/dev/zero of=file10 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.629831 s, 1.7 GB/s [root@localhost files]# [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# fio --name=pat_test --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --do_verify=1 --verify=pattern --verify_pattern=0x0a0b0c0d01020304 --directory=/mnt/ost0/ --group_reporting pat_test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 3 (f=3): [_(1),V(3),_(2)][96.1%][r=310MiB/s][r=310 IOPS][eta 00m:06s] pat_test: (groupid=0, jobs=6): err= 0: pid=421459: Fri Jun 17 05:23:21 2022 read: IOPS=82, BW=82.9MiB/s (86.0MB/s)(12.0GiB/148164msec) slat (usec): min=114, max=2759.5k, avg=69707.12, stdev=179834.81 clat (usec): min=108, max=2768.7k, avg=211253.45, stdev=263999.58 lat (usec): min=287, max=2769.4k, avg=280967.39, stdev=272427.89 clat percentiles (usec): | 1.00th=[ 1582], 5.00th=[ 2474], 10.00th=[ 3130], | 20.00th=[ 4948], 30.00th=[ 19006], 40.00th=[ 41681], | 50.00th=[ 100140], 60.00th=[ 212861], 70.00th=[ 291505], | 80.00th=[ 396362], 90.00th=[ 534774], 95.00th=[ 683672], | 99.00th=[1233126], 99.50th=[1417675], 99.90th=[1837106], | 99.95th=[1954546], 99.99th=[2768241] bw ( KiB/s): min=20650, max=895722, per=100.00%, avg=99331.12, stdev=19452.86, samples=1437 iops : min= 16, max= 874, avg=92.96, stdev=19.04, samples=1437 lat (usec) : 250=0.02%, 500=0.02%, 750=0.02%, 1000=0.06% lat (msec) : 2=2.37%, 4=13.92%, 10=8.72%, 20=5.46%, 50=11.87% lat (msec) : 100=7.58%, 250=14.53%, 500=23.38%, 750=8.18%, 1000=2.12% lat (msec) : 2000=1.70%, >=2000=0.05% cpu : usr=0.57%, sys=1.13%, ctx=5751, majf=4, minf=6250 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=82.9MiB/s (86.0MB/s), 82.9MiB/s-82.9MiB/s (86.0MB/s-86.0MB/s), io=12.0GiB (12.9GB), run=148164-148164msec [root@localhost files]# [root@localhost files]# md5sum -c md5sum /mnt/ost0/pat_test.0.0: OK /mnt/ost0/pat_test.1.0: OK /mnt/ost0/pat_test.2.0: OK /mnt/ost0/pat_test.3.0: OK /mnt/ost0/pat_test.4.0: OK /mnt/ost0/pat_test.5.0: OK [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an unrecoverable error. An attempt was made to correct the error. Applications are unaffected. action: Determine if the device needs to be replaced, and clear the errors using 'zpool clear' or replace the device with 'zpool replace'. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-9P scan: resilvered 768M in 00:00:17 with 0 errors on Fri Jun 17 05:23:41 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 raidz2-0 DEGRADED 0 0 0 spare-0 DEGRADED 0 0 2.82K /root/test/test/files/file1 DEGRADED 0 0 54 too many errors /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 spare-9 DEGRADED 0 0 1.90K /root/test/test/files/file10 DEGRADED 0 0 978 too many errors /root/test/test/files/file20 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 spares /root/test/test/files/file19 INUSE currently in use /root/test/test/files/file20 INUSE currently in use errors: No known data errors [root@localhost files]# dd if=/dev/zero of=file3 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.508124 s, 2.1 GB/s [root@localhost files]# dd if=/dev/zero of=file13 bs=1M seek=4 count=1020 conv=notrunc oflag=direct 1020+0 records in 1020+0 records out 1069547520 bytes (1.1 GB, 1020 MiB) copied, 0.665489 s, 1.6 GB/s [root@localhost files]# [root@localhost files]# echo 3 > /proc/sys/vm/drop_caches [root@localhost files]# fio --name=pat_test --ioengine=libaio --fallocate=none --iodepth=4 --rw=read --bs=1M --size=2G --numjobs=6 --do_verify=1 --verify=pattern --verify_pattern=0x0a0b0c0d01020304 --directory=/mnt/ost0/ --group_reporting pat_test: (g=0): rw=read, bs=(R) 1024KiB-1024KiB, (W) 1024KiB-1024KiB, (T) 1024KiB-1024KiB, ioengine=libaio, iodepth=4 ... fio-3.19 Starting 6 processes Jobs: 1 (f=1): [_(2),V(1),_(3)][100.0%][r=62.9MiB/s][r=62 IOPS][eta 00m:00s] pat_test: (groupid=0, jobs=6): err= 0: pid=456574: Fri Jun 17 05:28:28 2022 read: IOPS=70, BW=70.1MiB/s (73.5MB/s)(12.0GiB/175360msec) slat (usec): min=118, max=957912, avg=81587.87, stdev=157177.89 clat (usec): min=92, max=960210, avg=246153.35, stdev=187213.88 lat (usec): min=287, max=960981, avg=327748.30, stdev=143874.91 clat percentiles (usec): | 1.00th=[ 1647], 5.00th=[ 2311], 10.00th=[ 2737], 20.00th=[ 3720], | 30.00th=[ 82314], 40.00th=[204473], 50.00th=[283116], 60.00th=[333448], | 70.00th=[371196], 80.00th=[413139], 90.00th=[476054], 95.00th=[530580], | 99.00th=[633340], 99.50th=[666895], 99.90th=[759170], 99.95th=[775947], | 99.99th=[960496] bw ( KiB/s): min=16564, max=314589, per=100.00%, avg=76142.65, stdev=6839.22, samples=1882 iops : min= 12, max= 303, avg=70.23, stdev= 6.70, samples=1882 lat (usec) : 100=0.01%, 500=0.05%, 1000=0.05% lat (msec) : 2=2.41%, 4=19.18%, 10=3.30%, 20=0.36%, 50=1.64% lat (msec) : 100=5.01%, 250=13.38%, 500=47.28%, 750=7.22%, 1000=0.12% cpu : usr=0.46%, sys=0.76%, ctx=4406, majf=0, minf=6222 IO depths : 1=0.1%, 2=0.1%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, >=64=0.0% submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0% issued rwts: total=12288,0,0,0 short=0,0,0,0 dropped=0,0,0,0 latency : target=0, window=0, percentile=100.00%, depth=4 Run status group 0 (all jobs): READ: bw=70.1MiB/s (73.5MB/s), 70.1MiB/s-70.1MiB/s (73.5MB/s-73.5MB/s), io=12.0GiB (12.9GB), run=175360-175360msec [root@localhost files]# [root@localhost files]# md5sum -c md5sum /mnt/ost0/pat_test.0.0: OK /mnt/ost0/pat_test.1.0: OK /mnt/ost0/pat_test.2.0: OK /mnt/ost0/pat_test.3.0: OK /mnt/ost0/pat_test.4.0: OK /mnt/ost0/pat_test.5.0: OK [root@localhost files]# [root@localhost files]# zpool status -v pool: pool-oss0 state: DEGRADED status: One or more devices has experienced an unrecoverable error. An attempt was made to correct the error. Applications are unaffected. action: Determine if the device needs to be replaced, and clear the errors using 'zpool clear' or replace the device with 'zpool replace'. see: https://openzfs.github.io/openzfs-docs/msg/ZFS-8000-9P scan: resilvered 768M in 00:00:17 with 0 errors on Fri Jun 17 05:23:41 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 DEGRADED 0 0 0 raidz2-0 DEGRADED 0 0 0 spare-0 DEGRADED 0 0 2.82K /root/test/test/files/file1 DEGRADED 0 0 54 too many errors /root/test/test/files/file19 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 DEGRADED 0 0 2.72K too many errors /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 spare-9 DEGRADED 0 0 1.90K /root/test/test/files/file10 DEGRADED 0 0 978 too many errors /root/test/test/files/file20 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 DEGRADED 0 0 2.80K too many errors /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 spares /root/test/test/files/file19 INUSE currently in use /root/test/test/files/file20 INUSE currently in use errors: No known data errors [root@localhost files]# zpool clear pool-oss0 [root@localhost files]# zpool status -v pool: pool-oss0 state: ONLINE scan: resilvered 768M in 00:00:17 with 0 errors on Fri Jun 17 05:23:41 2022 config: NAME STATE READ WRITE CKSUM pool-oss0 ONLINE 0 0 0 raidz2-0 ONLINE 0 0 0 /root/test/test/files/file1 ONLINE 0 0 0 /root/test/test/files/file2 ONLINE 0 0 0 /root/test/test/files/file3 ONLINE 0 0 0 /root/test/test/files/file4 ONLINE 0 0 0 /root/test/test/files/file5 ONLINE 0 0 0 /root/test/test/files/file6 ONLINE 0 0 0 /root/test/test/files/file7 ONLINE 0 0 0 /root/test/test/files/file8 ONLINE 0 0 0 /root/test/test/files/file9 ONLINE 0 0 0 /root/test/test/files/file10 ONLINE 0 0 0 /root/test/test/files/file11 ONLINE 0 0 0 /root/test/test/files/file12 ONLINE 0 0 0 /root/test/test/files/file13 ONLINE 0 0 0 /root/test/test/files/file14 ONLINE 0 0 0 /root/test/test/files/file15 ONLINE 0 0 0 /root/test/test/files/file16 ONLINE 0 0 0 /root/test/test/files/file17 ONLINE 0 0 0 /root/test/test/files/file18 ONLINE 0 0 0 spares /root/test/test/files/file19 AVAIL /root/test/test/files/file20 AVAIL errors: No known data errors [root@localhost files]#