[WIP] Reset user accounting when raw receiving a snapshot

module/zfs/dmu_objset.c

@@ -782,6 +782,8 @@ dmu_objset_own(const char *name, dmu_objset_type_t type,
 	 */
 	if (!readonly && !dp->dp_spa->spa_claiming &&
 	    (ds->ds_dir->dd_crypto_obj == 0 || decrypt)) {
+		cmn_err(CE_NOTE, "upgrade, dn_type_uu: %llu",
+		    (*osp)->os_userused_dnode.dnh_dnode->dn_type);
 		if (dmu_objset_userobjspace_upgradable(*osp) ||
 		    dmu_objset_projectquota_upgradable(*osp)) {
 			dmu_objset_id_quota_upgrade(*osp);
@@ -1078,18 +1080,13 @@ dmu_objset_create_impl_dnstats(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 	 */
 	if (dmu_objset_userused_enabled(os) &&
 	    (!os->os_encrypted || !dmu_objset_is_receiving(os))) {
-		os->os_phys->os_flags |= OBJSET_FLAG_USERACCOUNTING_COMPLETE;
 		if (dmu_objset_userobjused_enabled(os)) {
 			ds->ds_feature_activation[
 			    SPA_FEATURE_USEROBJ_ACCOUNTING] = (void *)B_TRUE;
-			os->os_phys->os_flags |=
-			    OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE;
 		}
 		if (dmu_objset_projectquota_enabled(os)) {
 			ds->ds_feature_activation[
 			    SPA_FEATURE_PROJECT_QUOTA] = (void *)B_TRUE;
-			os->os_phys->os_flags |=
-			    OBJSET_FLAG_PROJECTQUOTA_COMPLETE;
 		}
 		os->os_flags = os->os_phys->os_flags;
 	}
@@ -2065,6 +2062,7 @@ dmu_objset_do_userquota_updates_prep(objset_t *os, dmu_tx_t *tx)
 
 	/* Allocate the user/group/project used objects if necessary. */
 	if (DMU_USERUSED_DNODE(os)->dn_type == DMU_OT_NONE) {
+		cmn_err(CE_NOTE, "create obj");
 		VERIFY0(zap_create_claim(os,
 		    DMU_USERUSED_OBJECT,
 		    DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx));

module/zfs/dnode.c

@@ -1605,7 +1605,11 @@ dnode_rele_and_unlock(dnode_t *dn, void *tag, boolean_t evicting)
 	dmu_buf_impl_t *db = dn->dn_dbuf;
 	dnode_handle_t *dnh = dn->dn_handle;
 
-	refs = zfs_refcount_remove(&dn->dn_holds, tag);
+	if (zfs_refcount_count(&dn->dn_holds) > 0) {
+		refs = zfs_refcount_remove(&dn->dn_holds, tag);
+	} else {
+		refs = 0;
+	}
 	if (refs == 0)
 		cv_broadcast(&dn->dn_nodnholds);
 	mutex_exit(&dn->dn_mtx);

module/zfs/dnode_sync.c

@@ -595,10 +595,11 @@ dnode_sync_free(dnode_t *dn, dmu_tx_t *tx)
 	ASSERT(dn->dn_type != DMU_OT_NONE);
 
 	ASSERT(dn->dn_free_txg > 0);
-	if (dn->dn_allocated_txg != dn->dn_free_txg)
+	if (dn->dn_allocated_txg != dn->dn_free_txg && &dn->dn_dbuf->db != NULL)
 		dmu_buf_will_dirty(&dn->dn_dbuf->db, tx);
 	bzero(dn->dn_phys, sizeof (dnode_phys_t) * dn->dn_num_slots);
-	dnode_free_interior_slots(dn);
+	if (&dn->dn_dbuf->db != NULL)
+		dnode_free_interior_slots(dn);
 
 	mutex_enter(&dn->dn_mtx);
 	dn->dn_type = DMU_OT_NONE;

module/zfs/dsl_crypt.c

@@ -2007,14 +2007,6 @@ dsl_crypto_recv_raw_objset_check(dsl_dataset_t *ds, dsl_dataset_t *fromds,
 	if (ret != 0)
 		return (ret);
 
-	/*
-	 * Useraccounting is not portable and must be done with the keys loaded.
-	 * Therefore, whenever we do any kind of receive the useraccounting
-	 * must not be present.
-	 */
-	ASSERT0(os->os_flags & OBJSET_FLAG_USERACCOUNTING_COMPLETE);
-	ASSERT0(os->os_flags & OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE);
-
 	mdn = DMU_META_DNODE(os);
 
 	/*
@@ -2106,8 +2098,23 @@ dsl_crypto_recv_raw_objset_sync(dsl_dataset_t *ds, dmu_objset_type_t ostype,
 	arc_release(os->os_phys_buf, &os->os_phys_buf);
 	bcopy(portable_mac, os->os_phys->os_portable_mac, ZIO_OBJSET_MAC_LEN);
 	bzero(os->os_phys->os_local_mac, ZIO_OBJSET_MAC_LEN);
+
+	/*
+	 * Needed for triggering recalculation of metadata
+	 */
+	os->os_phys->os_flags &= ~OBJSET_FLAG_USERACCOUNTING_COMPLETE;
+	os->os_phys->os_flags &= ~OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE;
+	os->os_phys->os_flags &= ~OBJSET_FLAG_PROJECTQUOTA_COMPLETE;
+	os->os_flags = os->os_phys->os_flags;
 	os->os_next_write_raw[tx->tx_txg & TXG_MASK] = B_TRUE;
 
+	/*
+	 * Needed for zeroing out the local_mac
+	 */
+	dmu_object_free(os, DMU_USERUSED_OBJECT, tx);
+	dmu_object_free(os, DMU_GROUPUSED_OBJECT, tx);
+	dmu_object_free(os, DMU_PROJECTUSED_OBJECT, tx);
+
 	/* set metadnode compression and checksum */
 	mdn->dn_compress = compress;
 	mdn->dn_checksum = checksum;
@@ -2131,6 +2138,8 @@ dsl_crypto_recv_raw_objset_sync(dsl_dataset_t *ds, dmu_objset_type_t ostype,
 		/* dsl_dataset_sync_done will drop this reference. */
 		dmu_buf_add_ref(ds->ds_dbuf, ds);
 		dsl_dataset_sync_done(ds, tx);
+
+		dsl_dataset_create_key_mapping(ds);
 	}
 }
 

module/zfs/dsl_pool.c

@@ -682,6 +682,8 @@ dsl_pool_sync(dsl_pool_t *dp, uint64_t txg)
 	for (ds = list_head(&synced_datasets); ds != NULL;
 	    ds = list_next(&synced_datasets, ds)) {
 		dmu_objset_sync_done(ds->ds_objset, tx);
+		cmn_err(CE_NOTE, "os_p_flags: %llu",
+		    ds->ds_objset->os_phys->os_flags);
 	}
 	taskq_wait(dp->dp_sync_taskq);
 

tests/zfs-tests/tests/functional/userquota/userspace_send_encrypted.ksh

@@ -0,0 +1,110 @@
+#!/bin/ksh -p
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+
+#
+# Copyright 2020, George Amanakis <[email protected]>. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+. $STF_SUITE/tests/functional/userquota/userquota_common.kshlib
+
+#
+# DESCRIPTION:
+# Sending raw encrypted datasets back to the source dataset succeeds.
+#
+#
+# STRATEGY:
+# 1. Create encrypted source dataset, set userquota and write a file
+# 2. Create base and an additional snapshot (s1)
+# 3. Unmount the source dataset
+# 4. Raw send the base snapshot to a new target dataset
+# 5. Raw send incrementally the s1 snapshot to the new target dataset
+# 6. Mount both source and target datasets
+# 7. Verify encrypted datasets support 'zfs userspace' and 'zfs groupspace'
+#	and the accounting is done correctly
+#
+
+function cleanup
+{
+	destroy_pool $POOLNAME
+	rm -f $FILEDEV
+}
+
+function log_must_unsupported
+{
+	log_must_retry "unsupported" 3 "$@"
+	(( $? != 0 )) && log_fail
+}
+
+log_onexit cleanup
+
+FILEDEV="$TEST_BASE_DIR/userspace_encrypted"
+POOLNAME="testpool$$"
+ENC_SOURCE="$POOLNAME/source"
+ENC_TARGET="$POOLNAME/target"
+
+log_assert "Sending raw encrypted datasets back to the source dataset succeeds."
+
+# Setup
+truncate -s 200m $FILEDEV
+log_must zpool create -o feature@encryption=enabled $POOLNAME \
+	$FILEDEV
+
+# Create encrypted source dataset
+log_must eval "echo 'password' | zfs create -o encryption=on" \
+	"-o keyformat=passphrase -o keylocation=prompt " \
+	"$ENC_SOURCE"
+
+# Set user quota and write file
+log_must zfs set userquota@$QUSER1=50m $ENC_SOURCE
+mkmount_writable $ENC_SOURCE
+mntpnt=$(get_prop mountpoint $ENC_SOURCE)
+log_must user_run $QUSER1 mkfile 20m /$mntpnt/file
+sync
+
+# Snapshot, raw send to new dataset
+log_must zfs snap $ENC_SOURCE@base
+log_must zfs snap $ENC_SOURCE@s1
+log_must zfs umount $ENC_SOURCE
+log_must eval "zfs send -w $ENC_SOURCE@base | zfs recv " \
+	"$ENC_TARGET"
+
+log_must eval "zfs send -w -i @base $ENC_SOURCE@s1 | zfs recv " \
+	"$ENC_TARGET"
+
+log_must zfs destroy $ENC_SOURCE@s1
+log_must eval "zfs send -w -i @base $ENC_TARGET@s1 | zfs recv " \
+	"$ENC_SOURCE"
+
+#  Mount encrypted datasets and verify they support 'zfs userspace' and
+# 'zfs groupspace' and the accounting is done correctly
+log_must zfs mount $ENC_SOURCE
+log_must eval "echo password | zfs load-key $ENC_TARGET"
+log_must zfs mount $ENC_TARGET
+sync
+
+sleep 5
+
+src_uspace=$(( $(zfs userspace -Hp $ENC_SOURCE | grep $QUSER1 | \
+	awk '{print $4}')/1024/1024))
+tgt_uspace=$(( $(zfs userspace -Hp $ENC_TARGET | grep $QUSER1 | \
+	awk '{print $4}')/1024/1024))
+log_must test "$src_uspace" -eq "$tgt_uspace"
+
+src_uquota=$(zfs userspace -Hp $ENC_SOURCE | grep $QUSER1 | awk '{print $5}')
+tgt_uquota=$(zfs userspace -Hp $ENC_TARGET | grep $QUSER1 | awk '{print $5}')
+log_must test "$src_uquota" -eq "$tgt_uquota"
+
+# Cleanup
+cleanup
+
+log_pass "Sending raw encrypted datasets back to the source dataset succeeds."