another rework of getzfsvfs / getzfsvfs_impl code

This change is designed to account for yet another difference between
illumos and FreeBSD VFS.  In FreeBSD a filesystem driver is supposed to
clean up mnt_data in its VFS_UNMOUNT method because it's the last call
into the driver before a struct mount object is destroyed.  The VFS
drains all references to the object before destroying it, but for the
driver it's already as good as gone.
In contrast, illumos VFS provides another method, VFS_FREEVFS, that is
called when all references are drained.  So, the driver can keep its
data after VFS_UNMOUNT and clean it up in VFS_FREEVFS after all
references are gone. This is what ZFS does on illumos.
So there a reference to a filesystem is sufficient to guarantee that the
ZFS specific data, aka zfsvfs_t, stays around (even if the filesystem
gets unmounted).  In FreeBSD we need to vfs_busy the filesystem to get
the same guarantee.  vfs_ref guarantees only that the struct mount is
kept.

The following rules should be observed in getzfsvfs / getzfsvfs_impl on
FreeBSD:
- if we need access to zfsvfs_t then we must use vfs_busy
- if only we need to access struct mount (aka vfs_t), then vfs_ref is
  enough
- when illumos code actually needs only the vfs_t, they still can pass
  the zfsvfs_t and get the vfs_t from it;  that can work in FreeBSD if
  the filesystem is busied, but when it's just referenced then we have
  to pass the vfs_t explicitly
- we cannot call vfs_busy while holding a dataset because that creates a
  LOR with dp_config_rwlock

As a result:
- getzfsvfs_impl now only references the filesystem, same as in illumos,
  but unlike illumos it has to return the vfs_t
- the consumers are updated to account for the change
- getzfsvfs busies the filesystem (and drops the reference from
  getzfsvfs_impl)

Also, zfs_unmount_snap() now gets a busied a filesystem, references it
and then unbusies it essentially reverting actions done in getzfsvfs.
This is needed because the code may perform some checks that require the
zfsvfs_t.  So, those are done before the unbusying.

MFC after:	2 weeks

sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_ioctl.h

@@ -426,7 +426,11 @@ extern int zfs_secpolicy_destroy_perms(const char *, cred_t *);
 extern int zfs_busy(void);
 extern void zfs_unmount_snap(const char *);
 extern void zfs_destroy_unmount_origin(const char *);
+#ifdef illumos
 extern int getzfsvfs_impl(struct objset *, struct zfsvfs **);
+#else
+extern int getzfsvfs_impl(struct objset *, vfs_t **);
+#endif
 extern int getzfsvfs(const char *, struct zfsvfs **);
 
 /*

sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zcp_get.c

@@ -226,7 +226,9 @@ get_temporary_prop(dsl_dataset_t *ds, zfs_prop_t zfs_prop, uint64_t *val,
 	return (0);
 #else
 	int error;
+#ifdef illumos
 	zfsvfs_t *zfvp;
+#endif
 	vfs_t *vfsp;
 	objset_t *os;
 	uint64_t tmp = *val;
@@ -235,12 +237,12 @@ get_temporary_prop(dsl_dataset_t *ds, zfs_prop_t zfs_prop, uint64_t *val,
 	if (error != 0)
 		return (error);
 
-	error = getzfsvfs_impl(os, &zfvp);
+	error = getzfsvfs_impl(os, &vfsp);
 	if (error != 0)
 		return (error);
-
+#ifdef illumos
 	vfsp = zfvp->z_vfs;
-
+#endif
 	switch (zfs_prop) {
 	case ZFS_PROP_ATIME:
 		if (vfs_optionisset(vfsp, MNTOPT_NOATIME, NULL))

sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c

@@ -1448,9 +1448,9 @@ put_nvlist(zfs_cmd_t *zc, nvlist_t *nvl)
 }
 
 int
-getzfsvfs_impl(objset_t *os, zfsvfs_t **zfvp)
+getzfsvfs_impl(objset_t *os, vfs_t **vfsp)
 {
-	vfs_t *vfsp;
+	zfsvfs_t *zfvp;
 	int error = 0;
 
 	if (dmu_objset_type(os) != DMU_OST_ZFS) {
@@ -1458,9 +1458,10 @@ getzfsvfs_impl(objset_t *os, zfsvfs_t **zfvp)
 	}
 
 	mutex_enter(&os->os_user_ptr_lock);
-	*zfvp = dmu_objset_get_user(os);
-	if (*zfvp) {
-		vfs_ref((*zfvp)->z_vfs);
+	zfvp = dmu_objset_get_user(os);
+	if (zfvp) {
+		*vfsp = zfvp->z_vfs;
+		vfs_ref(zfvp->z_vfs);
 	} else {
 		error = SET_ERROR(ESRCH);
 	}
@@ -1468,57 +1469,31 @@ getzfsvfs_impl(objset_t *os, zfsvfs_t **zfvp)
 	return (error);
 }
 
-#ifdef illumos
 int
 getzfsvfs(const char *dsname, zfsvfs_t **zfvp)
 {
 	objset_t *os;
+	vfs_t *vfsp;
 	int error;
 
 	error = dmu_objset_hold(dsname, FTAG, &os);
 	if (error != 0)
 		return (error);
-
-	error = getzfsvfs_impl(os, zfvp);
+	error = getzfsvfs_impl(os, &vfsp);
 	dmu_objset_rele(os, FTAG);
-	return (error);
-}
-
-#else
-
-static int
-getzfsvfs_ref(const char *dsname, zfsvfs_t **zfvp)
-{
-	objset_t *os;
-	int error;
-
-	error = dmu_objset_hold(dsname, FTAG, &os);
 	if (error != 0)
 		return (error);
 
-	error = getzfsvfs_impl(os, zfvp);
-	dmu_objset_rele(os, FTAG);
-	return (error);
-}
-
-int
-getzfsvfs(const char *dsname, zfsvfs_t **zfvp)
-{
-	objset_t *os;
-	int error;
-
-	error = getzfsvfs_ref(dsname, zfvp);
-	if (error != 0)
-		return (error);
-	error = vfs_busy((*zfvp)->z_vfs, 0);
-	vfs_rel((*zfvp)->z_vfs);
+	error = vfs_busy(vfsp, 0);
+	vfs_rel(vfsp);
 	if (error != 0) {
 		*zfvp = NULL;
 		error = SET_ERROR(ESRCH);
+	} else {
+		*zfvp = vfsp->vfs_data;
 	}
 	return (error);
 }
-#endif
 
 /*
  * Find a zfsvfs_t for a mounted filesystem, or create our own, in which
@@ -3597,7 +3572,7 @@ zfs_unmount_snap(const char *snapname)
 	if (strchr(snapname, '@') == NULL)
 		return;
 
-	int err = getzfsvfs_ref(snapname, &zfsvfs);
+	int err = getzfsvfs(snapname, &zfsvfs);
 	if (err != 0) {
 		ASSERT3P(zfsvfs, ==, NULL);
 		return;
@@ -3619,6 +3594,8 @@ zfs_unmount_snap(const char *snapname)
 #ifdef illumos
 	(void) dounmount(vfsp, MS_FORCE, kcred);
 #else
+	vfs_ref(vfsp);
+	vfs_unbusy(vfsp);
 	(void) dounmount(vfsp, MS_FORCE, curthread);
 #endif
 }