Skip to content

Commit

Permalink
Fix ENOSPC when unlinking multiple files from full pool
Browse files Browse the repository at this point in the history
When unlinking multiple files from a pool at 100% capacity, it was
possible for ENOSPC to be returned after the first unlink.  e.g.

    rm -f /mnt/fs/test1.0.0 /mnt/fs/test1.1.0 /mnt/fs/test1.2.0
    rm: cannot remove '/mnt/fs/test1.1.0': No space left on device
    rm: cannot remove '/mnt/fs/test1.2.0': No space left on device

After waiting for the pending deferred frees from the first unlink to
be processed the remaining files can then be unlinked.  This is caused
by the quota limit in dsl_dir_tempreserve_impl() being temporarily
decreased to the allocatable pool capacity less any deferred free
space.

This is resolved using the existing mechanism of returning ERESTART
when over quota as long as we know enough space will shortly be
available after processing the pending deferred frees.

Reviewed-by: Alexander Motin <[email protected]>
Reviewed-by: Ryan Moeller <[email protected]>
Reviewed-by: Tony Hutter <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>
Closes #13172
  • Loading branch information
behlendorf authored Mar 8, 2022
1 parent 39a4daf commit 6df4316
Show file tree
Hide file tree
Showing 6 changed files with 84 additions and 8 deletions.
1 change: 1 addition & 0 deletions include/sys/dsl_pool.h
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ int dsl_pool_sync_context(dsl_pool_t *dp);
uint64_t dsl_pool_adjustedsize(dsl_pool_t *dp, zfs_space_check_t slop_policy);
uint64_t dsl_pool_unreserved_space(dsl_pool_t *dp,
zfs_space_check_t slop_policy);
uint64_t dsl_pool_deferred_space(dsl_pool_t *dp);
void dsl_pool_wrlog_count(dsl_pool_t *dp, int64_t size, uint64_t txg);
boolean_t dsl_pool_wrlog_over_max(dsl_pool_t *dp);
void dsl_pool_dirty_space(dsl_pool_t *dp, int64_t space, dmu_tx_t *tx);
Expand Down
20 changes: 14 additions & 6 deletions module/zfs/dsl_dir.c
Original file line number Diff line number Diff line change
Expand Up @@ -1334,21 +1334,29 @@ dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree,
/*
* If they are requesting more space, and our current estimate
* is over quota, they get to try again unless the actual
* on-disk is over quota and there are no pending changes (which
* may free up space for us).
* on-disk is over quota and there are no pending changes
* or deferred frees (which may free up space for us).
*/
if (used_on_disk + est_inflight >= quota) {
if (est_inflight > 0 || used_on_disk < quota ||
(retval == ENOSPC && used_on_disk < quota))
retval = ERESTART;
if (est_inflight > 0 || used_on_disk < quota) {
retval = SET_ERROR(ERESTART);
} else {
ASSERT3U(used_on_disk, >=, quota);

if (retval == ENOSPC && (used_on_disk - quota) <
dsl_pool_deferred_space(dd->dd_pool)) {
retval = SET_ERROR(ERESTART);
}
}

dprintf_dd(dd, "failing: used=%lluK inflight = %lluK "
"quota=%lluK tr=%lluK err=%d\n",
(u_longlong_t)used_on_disk>>10,
(u_longlong_t)est_inflight>>10,
(u_longlong_t)quota>>10, (u_longlong_t)asize>>10, retval);
mutex_exit(&dd->dd_lock);
DMU_TX_STAT_BUMP(dmu_tx_quota);
return (SET_ERROR(retval));
return (retval);
}

/* We need to up our estimated delta before dropping dd_lock */
Expand Down
6 changes: 6 additions & 0 deletions module/zfs/dsl_pool.c
Original file line number Diff line number Diff line change
Expand Up @@ -950,6 +950,12 @@ dsl_pool_unreserved_space(dsl_pool_t *dp, zfs_space_check_t slop_policy)
return (quota);
}

uint64_t
dsl_pool_deferred_space(dsl_pool_t *dp)
{
return (metaslab_class_get_deferred(spa_normal_class(dp->dp_spa)));
}

boolean_t
dsl_pool_need_dirty_delay(dsl_pool_t *dp)
{
Expand Down
2 changes: 1 addition & 1 deletion tests/runfiles/common.run
Original file line number Diff line number Diff line change
Expand Up @@ -694,7 +694,7 @@ tags = ['functional', 'nestedfs']

[tests/functional/no_space]
tests = ['enospc_001_pos', 'enospc_002_pos', 'enospc_003_pos',
'enospc_df']
'enospc_df', 'enospc_rm']
tags = ['functional', 'no_space']

[tests/functional/nopwrite]
Expand Down
3 changes: 2 additions & 1 deletion tests/zfs-tests/tests/functional/no_space/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ dist_pkgdata_SCRIPTS = \
enospc_001_pos.ksh \
enospc_002_pos.ksh \
enospc_003_pos.ksh \
enospc_df.ksh
enospc_df.ksh \
enospc_rm.ksh

dist_pkgdata_DATA = \
enospc.cfg
60 changes: 60 additions & 0 deletions tests/zfs-tests/tests/functional/no_space/enospc_rm.ksh
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/bin/ksh -p
#
# CDDL HEADER START
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
# CDDL HEADER END
#

#
# Copyright (c) 2014, 2016 by Delphix. All rights reserved.
# Copyright (c) 2022 by Lawrence Livermore National Security, LLC.
#

. $STF_SUITE/include/libtest.shlib
. $STF_SUITE/tests/functional/no_space/enospc.cfg

#
# DESCRIPTION:
# After filling a filesystem, verify the contents can be removed
# without encountering an ENOSPC error.
#

verify_runnable "both"

function cleanup
{
destroy_pool $TESTPOOL
log_must rm -f $all_vdevs
}

log_onexit cleanup

log_assert "Files can be removed from full file system."

all_vdevs=$(echo $TEST_BASE_DIR/file.{01..12})

log_must truncate -s $MINVDEVSIZE $all_vdevs

log_must zpool create -f $TESTPOOL draid2:8d:2s $all_vdevs
log_must zfs create $TESTPOOL/$TESTFS
log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS
log_must zfs set compression=off $TESTPOOL/$TESTFS

log_note "Writing files until ENOSPC."
log_mustnot_expect "No space left on device" fio --name=test \
--fallocate=none --rw=write --bs=1M --size=1G --numjobs=4 \
--sync=1 --directory=$TESTDIR/ --group_reporting

log_must rm $TESTDIR/test.*
log_must test -z "$(ls -A $TESTDIR)"

log_pass "All files removed without error"

0 comments on commit 6df4316

Please sign in to comment.