Test case: zpool_upgrade_007_pos

[behlendorf]

System information

Type	Version/Name
Distribution Name	all
Distribution Version	all
Linux Kernel	all
Architecture	all
ZFS Version	0.7.0-rc4
SPL Version	0.7.0-rc4

Describe the problem you're observing

Occasional failure of test case zpool_upgrade_007_pos

Describe how to reproduce the problem

Reproducible occasionally by the automated testing and locally.

Include any warning/errors/backtraces from the system logs

[ 8394.810713] VERIFY3(newds == os->os_dsl_dataset) failed (ffff88000d383000 == ffff88004d5da800)
[ 8394.813104] PANIC at dmu_objset.c:726:dmu_objset_refresh_ownership()
[ 8394.813929] Showing stack for process 24000
[ 8394.813939] CPU: 2 PID: 24000 Comm: lt-zfs Tainted: P           OE  ------------   3.10.0-514.16.1.el7.x86_64 #1
[ 8394.813941] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS Ubuntu-1.8.2-1ubuntu1 04/01/2014
[ 8394.813943]  ffffffffa0904c16 00000000ce97c6ab ffff88004ddafa10 ffffffff81686ac3
[ 8394.813947]  ffff88004ddafa20 ffffffffa0687e34 ffff88004ddafba8 ffffffffa0687eff
[ 8394.813950]  0000000000000000 0000000000000030 ffff88004ddafbb8 ffff88004ddafb58
[ 8394.813954] Call Trace:
[ 8394.813979]  [<ffffffff81686ac3>] dump_stack+0x19/0x1b
[ 8394.814004]  [<ffffffffa0687e34>] spl_dumpstack+0x44/0x50 [spl]
[ 8394.814011]  [<ffffffffa0687eff>] spl_panic+0xbf/0xf0 [spl]
[ 8394.814020]  [<ffffffff81698eb1>] ? ftrace_call+0x5/0x2f
[ 8394.814024]  [<ffffffff8168e305>] ? _raw_spin_unlock+0x5/0x30
[ 8394.814028]  [<ffffffff8168c3e5>] ? _cond_resched+0x5/0x50
[ 8394.814121]  [<ffffffffa07d4714>] dmu_objset_refresh_ownership+0x104/0x1e0 [zfs]
[ 8394.814136]  [<ffffffff810b5a55>] ? up_write+0x5/0x20
[ 8394.814140]  [<ffffffff8168c3e5>] ? _cond_resched+0x5/0x50
[ 8394.814143]  [<ffffffff8168b312>] ? down_write+0x12/0x30
[ 8394.814146]  [<ffffffff81698eb1>] ? ftrace_call+0x5/0x2f
[ 8394.814169]  [<ffffffffa07d0d94>] ? dmu_objset_evict_dbufs+0x194/0x1b0 [zfs]
[ 8394.814175]  [<ffffffffa067fe0a>] ? spl_kmem_free+0x2a/0x40 [spl]
[ 8394.814198]  [<ffffffffa07d4615>] ? dmu_objset_refresh_ownership+0x5/0x1e0 [zfs]
[ 8394.814251]  [<ffffffffa084e850>] zfs_ioc_userspace_upgrade+0xf0/0x120 [zfs]
[ 8394.814282]  [<ffffffffa084e765>] ? zfs_ioc_userspace_upgrade+0x5/0x120 [zfs]
[ 8394.814314]  [<ffffffffa0852ce8>] zfs_prop_set_special+0x338/0x550 [zfs]
[ 8394.814362]  [<ffffffffa0854d66>] zfs_set_prop_nvlist+0x136/0x3a0 [zfs]
[ 8394.814394]  [<ffffffffa085653f>] zfs_ioc_set_prop+0x10f/0x150 [zfs]
[ 8394.814428]  [<ffffffffa0856435>] ? zfs_ioc_set_prop+0x5/0x150 [zfs]
[ 8394.814471]  [<ffffffffa0853d56>] zfsdev_ioctl+0x506/0x550 [zfs]
[ 8394.814499]  [<ffffffffa0853850>] ? pool_status_check.part.24+0xb0/0xb0 [zfs]
[ 8394.814516]  [<ffffffff81212555>] do_vfs_ioctl+0x2d5/0x4b0
[ 8394.814520]  [<ffffffff812127d1>] SyS_ioctl+0xa1/0xc0
[ 8394.814526]  [<ffffffff81697189>] system_call_fastpath+0x16/0x1b

[loli10K]

@avg-I is proposing a patch to fix this issue on the OpenZFS developer ML (https://openzfs.topicbox.com/groups/developer/T9d8097b8fffd8bfa).

I've been running "zpool_upgrade_007_pos" on a local Debian builder for ~6 hours without triggering the VERIFY(), the fix looks promising, posting the patch here for completeness (modified to take into account encryption changes):

diff --git a/include/sys/dmu_objset.h b/include/sys/dmu_objset.h
index df9b1a7..3deb844 100644
--- a/include/sys/dmu_objset.h
+++ b/include/sys/dmu_objset.h
@@ -188,8 +188,8 @@ int dmu_objset_own(const char *name, dmu_objset_type_t type,
 int dmu_objset_own_obj(struct dsl_pool *dp, uint64_t obj,
     dmu_objset_type_t type, boolean_t readonly, boolean_t decrypt,
     void *tag, objset_t **osp);
-void dmu_objset_refresh_ownership(objset_t *os, boolean_t key_needed,
-    void *tag);
+void dmu_objset_refresh_ownership(struct dsl_dataset *ds,
+    struct dsl_dataset **newds, boolean_t decrypt, void *tag);
 void dmu_objset_rele(objset_t *os, void *tag);
 void dmu_objset_rele_flags(objset_t *os, boolean_t decrypt, void *tag);
 void dmu_objset_disown(objset_t *os, boolean_t decrypt, void *tag);
diff --git a/module/zfs/dmu_objset.c b/module/zfs/dmu_objset.c
index 0d9273f..daf78d8 100644
--- a/module/zfs/dmu_objset.c
+++ b/module/zfs/dmu_objset.c
@@ -785,24 +785,22 @@ dmu_objset_rele(objset_t *os, void *tag)
  * same name so that it can be partially torn down and reconstructed.
  */
 void
-dmu_objset_refresh_ownership(objset_t *os, boolean_t decrypt, void *tag)
+dmu_objset_refresh_ownership(dsl_dataset_t *ds, dsl_dataset_t **newds,
+    boolean_t decrypt, void *tag)
 {
 	dsl_pool_t *dp;
-	dsl_dataset_t *ds, *newds;
+	ds_hold_flags_t flags = (decrypt) ? DS_HOLD_FLAG_DECRYPT : 0;
 	char name[ZFS_MAX_DATASET_NAME_LEN];
 
-	ds = os->os_dsl_dataset;
 	VERIFY3P(ds, !=, NULL);
 	VERIFY3P(ds->ds_owner, ==, tag);
 	VERIFY(dsl_dataset_long_held(ds));
 
 	dsl_dataset_name(ds, name);
-	dp = dmu_objset_pool(os);
+	dp = ds->ds_dir->dd_pool;
 	dsl_pool_config_enter(dp, FTAG);
-	dmu_objset_disown(os, decrypt, tag);
-	VERIFY0(dsl_dataset_own(dp, name,
-	    (decrypt) ? DS_HOLD_FLAG_DECRYPT : 0, tag, &newds));
-	VERIFY3P(newds, ==, os->os_dsl_dataset);
+	dsl_dataset_disown(ds, flags, tag);
+	VERIFY0(dsl_dataset_own(dp, name, flags, tag, newds));
 	dsl_pool_config_exit(dp, FTAG);
 }
 
diff --git a/module/zfs/zfs_ioctl.c b/module/zfs/zfs_ioctl.c
index 6bee042..39ba60a 100644
--- a/module/zfs/zfs_ioctl.c
+++ b/module/zfs/zfs_ioctl.c
@@ -5290,14 +5290,14 @@ zfs_ioc_userspace_upgrade(zfs_cmd_t *zc)
 			 * objset needs to be closed & reopened (to grow the
 			 * objset_phys_t).  Suspend/resume the fs will do that.
 			 */
-			dsl_dataset_t *ds;
+			dsl_dataset_t *ds, *newds;
 
 			ds = dmu_objset_ds(zfsvfs->z_os);
 			error = zfs_suspend_fs(zfsvfs);
 			if (error == 0) {
-				dmu_objset_refresh_ownership(zfsvfs->z_os,
+				dmu_objset_refresh_ownership(ds, &newds,
 				    B_TRUE, zfsvfs);
-				error = zfs_resume_fs(zfsvfs, ds);
+				error = zfs_resume_fs(zfsvfs, newds);
 			}
 		}
 		if (error == 0)