-
Notifications
You must be signed in to change notification settings - Fork 1.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Too many vdev probe errors should suspend pool
Similar to what we saw in #16569, we need to consider that a replacing vdev should not be considered as fully contributing to the redundancy of a raidz vdev even though current IO has enough redundancy. When a failed vdev_probe() is faulting a disk, it now checks if that disk is required, and if so it suspends the pool until the admin can return the missing disks. Sponsored-by: Klara, Inc. Sponsored-by: Wasabi Technology, Inc. Signed-off-by: Don Brady <[email protected]>
- Loading branch information
Showing
4 changed files
with
169 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
145 changes: 145 additions & 0 deletions
145
tests/zfs-tests/tests/functional/fault/suspend_on_probe_errors.ksh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,145 @@ | ||
| #!/bin/ksh -p | ||
| # | ||
| # CDDL HEADER START | ||
| # | ||
| # The contents of this file are subject to the terms of the | ||
| # Common Development and Distribution License (the "License"). | ||
| # You may not use this file except in compliance with the License. | ||
| # | ||
| # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | ||
| # or https://opensource.org/licenses/CDDL-1.0. | ||
| # See the License for the specific language governing permissions | ||
| # and limitations under the License. | ||
| # | ||
| # When distributing Covered Code, include this CDDL HEADER in each | ||
| # file and include the License file at usr/src/OPENSOLARIS.LICENSE. | ||
| # If applicable, add the following below this CDDL HEADER, with the | ||
| # fields enclosed by brackets "[]" replaced with your own identifying | ||
| # information: Portions Copyright [yyyy] [name of copyright owner] | ||
| # | ||
| # CDDL HEADER END | ||
| # | ||
|
|
||
| # | ||
| # Copyright (c) 2024, Klara Inc. | ||
| # | ||
|
|
||
| . $STF_SUITE/include/libtest.shlib | ||
| . $STF_SUITE/include/blkdev.shlib | ||
|
|
||
| # | ||
| # DESCRIPTION: Verify that 4 disk removed from a raidz3 will suspend the pool | ||
| # | ||
| # STRATEGY: | ||
| # 1. Disable ZED -- this test is focused on vdev_probe errors | ||
| # 2. Create a raidz3 pool where 4 disks can be removed (i.e., using scsi_debug) | ||
| # 3. Add some data to it for a resilver workload | ||
| # 4. Replace one of the child vdevs to start a replacing vdev | ||
| # 5. During the resilver, remove 4 disks including one from the replacing vdev | ||
| # 6. Verify that the pool is suspended (it used to remain online) | ||
| # | ||
|
|
||
| DEV_SIZE_MB=1024 | ||
|
|
||
| FILE_VDEV_CNT=8 | ||
| FILE_VDEV_SIZ=256M | ||
|
|
||
| function cleanup | ||
| { | ||
| destroy_pool $TESTPOOL | ||
| unload_scsi_debug | ||
| rm -f $DATA_FILE | ||
| for i in {0..$((FILE_VDEV_CNT - 1))}; do | ||
| log_must rm -f "$TEST_BASE_DIR/dev-$i" | ||
| done | ||
| zed_start | ||
| } | ||
|
|
||
| log_onexit cleanup | ||
|
|
||
| log_assert "VDEV probe errors for more disks than parity should suspend a pool" | ||
|
|
||
| log_note "Stoping ZED process" | ||
| zed_stop | ||
| zpool events -c | ||
|
|
||
| # Make a debug device that we can "unplug" and loose 4 drives at once | ||
| unload_scsi_debug | ||
| load_scsi_debug $DEV_SIZE_MB 1 1 1 '512b' | ||
| sd=$(get_debug_device) | ||
|
|
||
| # Create 4 partitions that match the FILE_VDEV_SIZ | ||
| parted "/dev/${sd}" --script mklabel gpt | ||
| parted "/dev/${sd}" --script mkpart primary 0% 25% | ||
| parted "/dev/${sd}" --script mkpart primary 25% 50% | ||
| parted "/dev/${sd}" --script mkpart primary 50% 75% | ||
| parted "/dev/${sd}" --script mkpart primary 75% 100% | ||
| block_device_wait "/dev/${sd}" | ||
| blkdevs="/dev/${sd}1 /dev/${sd}2 /dev/${sd}3 /dev/${sd}4" | ||
|
|
||
| # Create 8 file vdevs | ||
| typeset -a filedevs | ||
| for i in {0..$((FILE_VDEV_CNT - 1))}; do | ||
| device=$TEST_BASE_DIR/dev-$i | ||
| log_must truncate -s $FILE_VDEV_SIZ $device | ||
| # Use all but the last one for pool create | ||
| if [[ $i -lt "7" ]]; then | ||
| filedevs[${#filedevs[*]}+1]=$device | ||
| fi | ||
| done | ||
|
|
||
| # Create a raidz-3 pool that we can pull 4 disks from | ||
| log_must zpool create -f $TESTPOOL raidz3 ${filedevs[@]} $blkdevs | ||
| sync_pool $TESTPOOL | ||
|
|
||
| # Add some data to the pool | ||
| log_must zfs create $TESTPOOL/fs | ||
| MNTPOINT="$(get_prop mountpoint $TESTPOOL/fs)" | ||
| SECONDS=0 | ||
| log_must fill_fs $MNTPOINT 3 200 32768 300 Z | ||
| log_note "fill_fs took $SECONDS seconds" | ||
| sync_pool $TESTPOOL | ||
|
|
||
| # Start a replacing vdev | ||
| log_must zpool replace -f $TESTPOOL /dev/${sd}4 $TEST_BASE_DIR/dev-7 | ||
|
|
||
| # Remove 4 disks all at once | ||
| log_must eval "echo offline > /sys/block/${sd}/device/state" | ||
|
|
||
| # Add some writes to drive the vdev probe errors | ||
| log_must dd if=/dev/urandom of=$MNTPOINT/writes bs=1M count=1 | ||
|
|
||
| # Wait until sync starts, and the pool suspends | ||
| log_note "waiting for pool to suspend" | ||
| typeset -i tries=10 | ||
| until [[ $(cat /proc/spl/kstat/zfs/$TESTPOOL/state) == "SUSPENDED" ]] ; do | ||
| if ((tries-- == 0)); then | ||
| zpool status -s | ||
| log_fail "UNEXPECTED -- pool did not suspend" | ||
| fi | ||
| sleep 1 | ||
| done | ||
|
|
||
| zpool status $TESTPOOL | ||
|
|
||
| # Put the missing disks back into service | ||
| log_must eval "echo running > /sys/block/$sd/device/state" | ||
|
|
||
| # Clear the vdev error states, which will reopen the vdevs and resume the pool | ||
| log_must zpool clear $TESTPOOL | ||
|
|
||
| # Wait until the pool resumes | ||
| log_note "waiting for pool to resume" | ||
| tries=10 | ||
| until [[ $(cat /proc/spl/kstat/zfs/$TESTPOOL/state) != "SUSPENDED" ]] ; do | ||
| if ((tries-- == 0)); then | ||
| log_fail "pool did not resume" | ||
| fi | ||
| sleep 1 | ||
| done | ||
|
|
||
| # Make sure a pool scrub comes back clean | ||
| log_must zpool scrub -w $TESTPOOL | ||
| log_must check_pool_status $pool "errors" "No known data errors" | ||
|
|
||
| log_pass "VDEV probe errors for more disks than parity should suspend a pool" |