Partial MFV (illumos-gate 13753:2aba784c276b)

2762 zpool command should have better support for feature flags

References:
https://www.illumos.org/issues/2762

MFC after:	2 weeks

Makefile.inc1

@@ -1260,7 +1260,7 @@ _prebuild_libs=	${_kerberos5_lib_libasn1} \
 		lib/ncurses/ncurses lib/ncurses/ncursesw \
 		lib/libopie lib/libpam ${_lib_libthr} \
 		lib/libradius lib/libsbuf lib/libtacplus \
-		${_cddl_lib_libumem} \
+		${_cddl_lib_libumem} ${_cddl_lib_libnvpair} \
 		lib/libutil ${_lib_libypclnt} lib/libz lib/msun \
 		${_secure_lib_libcrypto} ${_secure_lib_libssh} \
 		${_secure_lib_libssl}
@@ -1284,6 +1284,7 @@ lib/libopie__L lib/libtacplus__L: lib/libmd__L
 
 .if ${MK_CDDL} != "no"
 _cddl_lib_libumem= cddl/lib/libumem
+_cddl_lib_libnvpair= cddl/lib/libnvpair
 _cddl_lib= cddl/lib
 .endif
 

cddl/contrib/opensolaris/cmd/zpool/zpool.8

@@ -1636,21 +1636,22 @@ for unixtime
 .Op Fl v
 .Xc
 .Pp
-Displays all pools formatted using a different
+Displays pools which do not have all supported features enabled and pools
+formatted using a legacy
 .Tn ZFS
-pool on-disk version. Older versions can continue to be used, but some
+version number.
-features may not be available. These pools can be upgraded using
+These pools can continue to be used, but some features may not be available.
-.Qq Nm Cm upgrade Fl a .
+Use
-Pools that are formatted with a more recent version are also displayed,
+.Nm Cm upgrade Fl a
-although these pools will be inaccessible on the system.
+to enable all features on all pools.
 .Bl -tag -width indent
 .It Fl v
-Displays
+Displays legacy
 .Tn ZFS
-pool versions supported by the current software. The current
+versions supported by the current software.
-.Tn ZFS
+See
-pool version and all previous supported versions are displayed, along
+.Xr zpool-features.5
-with an explanation of the features provided with each version.
+for a description of feature flags features supported by the current software.
 .El
 .It Xo
 .Nm
@@ -1659,18 +1660,22 @@ with an explanation of the features provided with each version.
 .Fl a | Ar pool ...
 .Xc
 .Pp
-Upgrades the given pool to the latest on-disk pool version. Once this is done,
+Enables all supported features on the given pool.
-the pool will no longer be accessible on systems running older versions of the
+Once this is done, the pool will no longer be accessible on systems that do
-software.
+not support feature flags.
+See
+.Xr zpool-features.5
+for details on compatability with system sthat support feature flags, but do
+not support all features enabled on the pool.
 .Bl -tag -width indent
 .It Fl a
-Upgrades all pools.
+Enables all supported features on all pools.
 .It Fl V Ar version
-Upgrade to the specified version. If the
+Upgrade to the specified legacy version. If the
 .Fl V
-flag is not specified, the pool is upgraded to the most recent version. This
+flag is specified, no features will be enabled on the pool.
-option can only be used to increase the version number, and only up to the most
+This option can only be used to increase version number up to the last
-recent version supported by this software.
+supported legacy version number.
 .El
 .El
 .Sh EXAMPLES

cddl/contrib/opensolaris/cmd/zpool/zpool_main.c

@@ -389,6 +389,18 @@ print_vdev_tree(zpool_handle_t *zhp, const char *name, nvlist_t *nv, int indent,
 	}
 }
 
+static boolean_t
+prop_list_contains_feature(nvlist_t *proplist)
+{
+	nvpair_t *nvp;
+	for (nvp = nvlist_next_nvpair(proplist, NULL); NULL != nvp;
+	    nvp = nvlist_next_nvpair(proplist, nvp)) {
+		if (zpool_prop_feature(nvpair_name(nvp)))
+			return (B_TRUE);
+	}
+	return (B_FALSE);
+}
+
 /*
  * Add a property pair (name, string-value) into a property nvlist.
  */
@@ -412,12 +424,30 @@ add_prop_list(const char *propname, char *propval, nvlist_t **props,
 	proplist = *props;
 
 	if (poolprop) {
+		const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
+
 		if ((prop = zpool_name_to_prop(propname)) == ZPROP_INVAL &&
 		    !zpool_prop_feature(propname)) {
 			(void) fprintf(stderr, gettext("property '%s' is "
 			    "not a valid pool property\n"), propname);
 			return (2);
 		}
+
+		/*
+		 * feature@ properties and version should not be specified
+		 * at the same time.
+		 */
+		if ((prop == ZPROP_INVAL && zpool_prop_feature(propname) &&
+		    nvlist_exists(proplist, vname)) ||
+		    (prop == ZPOOL_PROP_VERSION &&
+		    prop_list_contains_feature(proplist))) {
+			(void) fprintf(stderr, gettext("'feature@' and "
+			    "'version' properties cannot be specified "
+			    "together\n"));
+			return (2);
+		}
+
+
 		if (zpool_prop_feature(propname))
 			normnm = propname;
 		else
@@ -1583,8 +1613,8 @@ show_import(nvlist_t *config)
 		break;
 
 	case ZPOOL_STATUS_VERSION_OLDER:
-		(void) printf(gettext(" status: The pool is formatted using an "
+		(void) printf(gettext(" status: The pool is formatted using a "
-		    "older on-disk version.\n"));
+		    "legacy on-disk version.\n"));
 		break;
 
 	case ZPOOL_STATUS_VERSION_NEWER:
@@ -1592,6 +1622,11 @@ show_import(nvlist_t *config)
 		    "incompatible version.\n"));
 		break;
 
+	case ZPOOL_STATUS_FEAT_DISABLED:
+		(void) printf(gettext(" status: Some supported features are "
+		    "not enabled on the pool.\n"));
+		break;
+
 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
 		(void) printf(gettext("status: The pool uses the following "
 		    "feature(s) not supported on this sytem:\n"));
@@ -1638,19 +1673,21 @@ show_import(nvlist_t *config)
 	 * Print out an action according to the overall state of the pool.
 	 */
 	if (vs->vs_state == VDEV_STATE_HEALTHY) {
-		if (reason == ZPOOL_STATUS_VERSION_OLDER)
+		if (reason == ZPOOL_STATUS_VERSION_OLDER ||
+		    reason == ZPOOL_STATUS_FEAT_DISABLED) {
 			(void) printf(gettext(" action: The pool can be "
 			    "imported using its name or numeric identifier, "
 			    "though\n\tsome features will not be available "
 			    "without an explicit 'zpool upgrade'.\n"));
-		else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH)
+		} else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
 			(void) printf(gettext(" action: The pool can be "
 			    "imported using its name or numeric "
 			    "identifier and\n\tthe '-f' flag.\n"));
-		else
+		} else {
 			(void) printf(gettext(" action: The pool can be "
 			    "imported using its name or numeric "
 			    "identifier.\n"));
+		}
 	} else if (vs->vs_state == VDEV_STATE_DEGRADED) {
 		(void) printf(gettext(" action: The pool can be imported "
 		    "despite missing or damaged devices.  The\n\tfault "
@@ -4108,12 +4145,13 @@ status_callback(zpool_handle_t *zhp, void *data)
 		break;
 
 	case ZPOOL_STATUS_VERSION_OLDER:
-		(void) printf(gettext("status: The pool is formatted using an "
+		(void) printf(gettext("status: The pool is formatted using a "
-		    "older on-disk format.  The pool can\n\tstill be used, but "
+		    "legacy on-disk format.  The pool can\n\tstill be used, "
-		    "some features are unavailable.\n"));
+		    "but some features are unavailable.\n"));
 		(void) printf(gettext("action: Upgrade the pool using 'zpool "
 		    "upgrade'.  Once this is done, the\n\tpool will no longer "
-		    "be accessible on older software versions.\n"));
+		    "be accessible on software that does not support feature\n"
+		    "\tflags.\n"));
 		break;
 
 	case ZPOOL_STATUS_VERSION_NEWER:
@@ -4125,6 +4163,16 @@ status_callback(zpool_handle_t *zhp, void *data)
 		    "backup.\n"));
 		break;
 
+	case ZPOOL_STATUS_FEAT_DISABLED:
+		(void) printf(gettext("status: Some supported features are not "
+		    "enabled on the pool. The pool can\n\tstill be used, but "
+		    "some features are unavailable.\n"));
+		(void) printf(gettext("action: Enable all features using "
+		    "'zpool upgrade'. Once this is done,\n\tthe pool may no "
+		    "longer be accessible by software that does not support\n\t"
+		    "the features. See zpool-features(5) for details.\n"));
+		break;
+
 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
 		(void) printf(gettext("status: The pool cannot be accessed on "
 		    "this system because it uses the\n\tfollowing feature(s) "
@@ -4354,9 +4402,7 @@ zpool_do_status(int argc, char **argv)
 }
 
 typedef struct upgrade_cbdata {
-	int	cb_all;
 	int	cb_first;
-	int	cb_newer;
 	char	cb_poolname[ZPOOL_MAXNAMELEN];
 	int	cb_argc;
 	uint64_t cb_version;
@@ -4388,56 +4434,157 @@ is_root_pool(zpool_handle_t *zhp)
 	return (poolname != NULL && strcmp(poolname, zpool_get_name(zhp)) == 0);
 }
 
+static int
+upgrade_version(zpool_handle_t *zhp, uint64_t version)
+{
+	int ret;
+	nvlist_t *config;
+	uint64_t oldversion;
+
+	config = zpool_get_config(zhp, NULL);
+	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
+	    &oldversion) == 0);
+
+	assert(SPA_VERSION_IS_SUPPORTED(oldversion));
+	assert(oldversion < version);
+
+	ret = zpool_upgrade(zhp, version);
+	if (ret != 0)
+		return (ret);
+
+	if (version >= SPA_VERSION_FEATURES) {
+		(void) printf(gettext("Successfully upgraded "
+		    "'%s' from version %llu to feature flags.\n"),
+		    zpool_get_name(zhp), oldversion);
+	} else {
+		(void) printf(gettext("Successfully upgraded "
+		    "'%s' from version %llu to version %llu.\n"),
+		    zpool_get_name(zhp), oldversion, version);
+	}
+
+	return (0);
+}
+
+static int
+upgrade_enable_all(zpool_handle_t *zhp, int *countp)
+{
+	int i, ret, count;
+	boolean_t firstff = B_TRUE;
+	nvlist_t *enabled = zpool_get_features(zhp);
+
+	count = 0;
+	for (i = 0; i < SPA_FEATURES; i++) {
+		const char *fname = spa_feature_table[i].fi_uname;
+		const char *fguid = spa_feature_table[i].fi_guid;
+		if (!nvlist_exists(enabled, fguid)) {
+			char *propname;
+			verify(-1 != asprintf(&propname, "feature@%s", fname));
+			ret = zpool_set_prop(zhp, propname,
+			    ZFS_FEATURE_ENABLED);
+			if (ret != 0) {
+				free(propname);
+				return (ret);
+			}
+			count++;
+
+			if (firstff) {
+				(void) printf(gettext("Enabled the "
+				    "following features on '%s':\n"),
+				    zpool_get_name(zhp));
+				firstff = B_FALSE;
+			}
+			(void) printf(gettext("  %s\n"), fname);
+			free(propname);
+		}
+	}
+
+	if (countp != NULL)
+		*countp = count;
+	return (0);
+}
+
 static int
 upgrade_cb(zpool_handle_t *zhp, void *arg)
 {
 	upgrade_cbdata_t *cbp = arg;
 	nvlist_t *config;
 	uint64_t version;
-	int ret = 0;
+	boolean_t printnl = B_FALSE;
+	int ret;
 
 	config = zpool_get_config(zhp, NULL);
 	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
 	    &version) == 0);
 
-	if (!cbp->cb_newer && SPA_VERSION_IS_SUPPORTED(version) &&
+	assert(SPA_VERSION_IS_SUPPORTED(version));
-	    version != SPA_VERSION) {
-		if (!cbp->cb_all) {
-			if (cbp->cb_first) {
-				(void) printf(gettext("The following pools are "
-				    "out of date, and can be upgraded.  After "
-				    "being\nupgraded, these pools will no "
-				    "longer be accessible by older software "
-				    "versions.\n\n"));
-				(void) printf(gettext("VER  POOL\n"));
-				(void) printf(gettext("---  ------------\n"));
-				cbp->cb_first = B_FALSE;
-			}
 
-			(void) printf("%2llu   %s\n", (u_longlong_t)version,
+	if (version < cbp->cb_version) {
-			    zpool_get_name(zhp));
+		cbp->cb_first = B_FALSE;
-		} else {
+		ret = upgrade_version(zhp, cbp->cb_version);
-			cbp->cb_first = B_FALSE;
+		if (ret != 0)
-			ret = zpool_upgrade(zhp, cbp->cb_version);
+			return (ret);
-			if (!ret) {
+#ifdef __FreeBSD__
-				(void) printf(gettext("Successfully upgraded "
+		if (cbp->cb_poolname[0] == '\0' &&
-				    "'%s'\n\n"), zpool_get_name(zhp));
+		    is_root_pool(zhp)) {
-				if (cbp->cb_poolname[0] == '\0' &&
+			(void) strlcpy(cbp->cb_poolname,
-				    is_root_pool(zhp)) {
+			    zpool_get_name(zhp),
-					(void) strlcpy(cbp->cb_poolname,
+			    sizeof(cbp->cb_poolname));
-					    zpool_get_name(zhp),
-					    sizeof(cbp->cb_poolname));
-				}
-			}
 		}
-	} else if (cbp->cb_newer && !SPA_VERSION_IS_SUPPORTED(version)) {
+#endif	/* ___FreeBSD__ */
-		assert(!cbp->cb_all);
+		printnl = B_TRUE;
 
+#ifdef illumos
+		/*
+		 * If they did "zpool upgrade -a", then we could
+		 * be doing ioctls to different pools.  We need
+		 * to log this history once to each pool, and bypass
+		 * the normal history logging that happens in main().
+		 */
+		(void) zpool_log_history(g_zfs, history_str);
+		log_history = B_FALSE;
+#endif
+	}
+
+	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
+		int count;
+		ret = upgrade_enable_all(zhp, &count);
+		if (ret != 0)
+			return (ret);
+
+		if (count > 0) {
+			cbp->cb_first = B_FALSE;
+			printnl = B_TRUE;
+		}
+	}
+
+	if (printnl) {
+		(void) printf(gettext("\n"));
+	}
+
+	return (0);
+}
+
+static int
+upgrade_list_older_cb(zpool_handle_t *zhp, void *arg)
+{
+	upgrade_cbdata_t *cbp = arg;
+	nvlist_t *config;
+	uint64_t version;
+
+	config = zpool_get_config(zhp, NULL);
+	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
+	    &version) == 0);
+
+	assert(SPA_VERSION_IS_SUPPORTED(version));
+
+	if (version < SPA_VERSION_FEATURES) {
 		if (cbp->cb_first) {
 			(void) printf(gettext("The following pools are "
-			    "formatted using an unsupported software version "
+			    "formatted with legacy version numbers and can\n"
-			    "and\ncannot be accessed on the current "
+			    "be upgraded to use feature flags.  After "
-			    "system.\n\n"));
+			    "being upgraded, these pools\nwill no "
+			    "longer be accessible by software that does not "
+			    "support feature\nflags.\n\n"));
 			(void) printf(gettext("VER  POOL\n"));
 			(void) printf(gettext("---  ------------\n"));
 			cbp->cb_first = B_FALSE;
@@ -4447,14 +4594,65 @@ upgrade_cb(zpool_handle_t *zhp, void *arg)
 		    zpool_get_name(zhp));
 	}
 
-	zpool_close(zhp);
+	return (0);
-	return (ret);
+}
+
+static int
+upgrade_list_disabled_cb(zpool_handle_t *zhp, void *arg)
+{
+	upgrade_cbdata_t *cbp = arg;
+	nvlist_t *config;
+	uint64_t version;
+
+	config = zpool_get_config(zhp, NULL);
+	verify(nvlist_lookup_uint64(config, ZPOOL_CONFIG_VERSION,
+	    &version) == 0);
+
+	if (version >= SPA_VERSION_FEATURES) {
+		int i;
+		boolean_t poolfirst = B_TRUE;
+		nvlist_t *enabled = zpool_get_features(zhp);
+
+		for (i = 0; i < SPA_FEATURES; i++) {
+			const char *fguid = spa_feature_table[i].fi_guid;
+			const char *fname = spa_feature_table[i].fi_uname;
+			if (!nvlist_exists(enabled, fguid)) {
+				if (cbp->cb_first) {
+					(void) printf(gettext("\nSome "
+					    "supported features are not "
+					    "enabled on the following pools. "
+					    "Once a\nfeature is enabled the "
+					    "pool may become incompatible with "
+					    "software\nthat does not support "
+					    "the feature. See "
+					    "zpool-features(5) for "
+					    "details.\n\n"));
+					(void) printf(gettext("POOL  "
+					    "FEATURE\n"));
+					(void) printf(gettext("------"
+					    "---------\n"));
+					cbp->cb_first = B_FALSE;
+				}
+
+				if (poolfirst) {
+					(void) printf(gettext("%s\n"),
+					    zpool_get_name(zhp));
+					poolfirst = B_FALSE;
+				}
+
+				(void) printf(gettext("      %s\n"), fname);
+			}
+		}
+	}
+
+	return (0);
 }
 
 /* ARGSUSED */
 static int
 upgrade_one(zpool_handle_t *zhp, void *data)
 {
+	boolean_t printnl = B_FALSE;
 	upgrade_cbdata_t *cbp = data;
 	uint64_t cur_version;
 	int ret;
@@ -4469,30 +4667,58 @@ upgrade_one(zpool_handle_t *zhp, void *data)
 	cur_version = zpool_get_prop_int(zhp, ZPOOL_PROP_VERSION, NULL);
 	if (cur_version > cbp->cb_version) {
 		(void) printf(gettext("Pool '%s' is already formatted "
-		    "using more current version '%llu'.\n"),
+		    "using more current version '%llu'.\n\n"),
 		    zpool_get_name(zhp), cur_version);
 		return (0);
 	}
-	if (cur_version == cbp->cb_version) {
+
+	if (cbp->cb_version != SPA_VERSION && cur_version == cbp->cb_version) {
 		(void) printf(gettext("Pool '%s' is already formatted "
-		    "using the current version.\n"), zpool_get_name(zhp));
+		    "using version %llu.\n\n"), zpool_get_name(zhp),
+		    cbp->cb_version);
 		return (0);
 	}
 
-	ret = zpool_upgrade(zhp, cbp->cb_version);
+	if (cur_version != cbp->cb_version) {
+		printnl = B_TRUE;
+		ret = upgrade_version(zhp, cbp->cb_version);
+		if (ret != 0) {
+#ifdef __FreeBSD__
+			if (cbp->cb_poolname[0] == '\0' &&
+			    is_root_pool(zhp)) {
+				(void) strlcpy(cbp->cb_poolname,
+				    zpool_get_name(zhp),
+				    sizeof(cbp->cb_poolname));
+			}
+#endif	/* ___FreeBSD__ */
+			return (ret);
+		}
+	}
 
-	if (!ret) {
+	if (cbp->cb_version >= SPA_VERSION_FEATURES) {
-		(void) printf(gettext("Successfully upgraded '%s' "
+		int count = 0;
-		    "from version %llu to version %llu\n\n"),
+		ret = upgrade_enable_all(zhp, &count);
-		    zpool_get_name(zhp), (u_longlong_t)cur_version,
+		if (ret != 0)
-		    (u_longlong_t)cbp->cb_version);
+			return (ret);
+
+		if (count != 0) {
+			printnl = B_TRUE;
+		} else if (cur_version == SPA_VERSION) {
+			(void) printf(gettext("Pool '%s' already has all "
+			    "supported features enabled.\n"),
+			    zpool_get_name(zhp));
+		}
 		if (cbp->cb_poolname[0] == '\0' && is_root_pool(zhp)) {
 			(void) strlcpy(cbp->cb_poolname, zpool_get_name(zhp),
 			    sizeof(cbp->cb_poolname));
 		}
 	}
 
-	return (ret != 0);
+	if (printnl) {
+		(void) printf(gettext("\n"));
+	}
+
+	return (0);
 }
 
 /*
@@ -4511,6 +4737,7 @@ zpool_do_upgrade(int argc, char **argv)
 	upgrade_cbdata_t cb = { 0 };
 	int ret = 0;
 	boolean_t showversions = B_FALSE;
+	boolean_t upgradeall = B_FALSE;
 	char *end;
 
 
@@ -4518,7 +4745,7 @@ zpool_do_upgrade(int argc, char **argv)
 	while ((c = getopt(argc, argv, ":avV:")) != -1) {
 		switch (c) {
 		case 'a':
-			cb.cb_all = B_TRUE;
+			upgradeall = B_TRUE;
 			break;
 		case 'v':
 			showversions = B_TRUE;
@@ -4551,19 +4778,19 @@ zpool_do_upgrade(int argc, char **argv)
 
 	if (cb.cb_version == 0) {
 		cb.cb_version = SPA_VERSION;
-	} else if (!cb.cb_all && argc == 0) {
+	} else if (!upgradeall && argc == 0) {
 		(void) fprintf(stderr, gettext("-V option is "
 		    "incompatible with other arguments\n"));
 		usage(B_FALSE);
 	}
 
 	if (showversions) {
-		if (cb.cb_all || argc != 0) {
+		if (upgradeall || argc != 0) {
 			(void) fprintf(stderr, gettext("-v option is "
 			    "incompatible with other arguments\n"));
 			usage(B_FALSE);
 		}
-	} else if (cb.cb_all) {
+	} else if (upgradeall) {
 		if (argc != 0) {
 			(void) fprintf(stderr, gettext("-a option should not "
 			    "be used along with a pool name\n"));
@@ -4573,9 +4800,25 @@ zpool_do_upgrade(int argc, char **argv)
 
 	(void) printf(gettext("This system supports ZFS pool feature "
 	    "flags.\n\n"));
-	cb.cb_first = B_TRUE;
 	if (showversions) {
-		(void) printf(gettext("The following versions are "
+		int i;
+
+		(void) printf(gettext("The following features are "
+		    "supported:\n\n"));
+		(void) printf(gettext("FEAT DESCRIPTION\n"));
+		(void) printf("----------------------------------------------"
+		    "---------------\n");
+		for (i = 0; i < SPA_FEATURES; i++) {
+			zfeature_info_t *fi = &spa_feature_table[i];
+			const char *ro = fi->fi_can_readonly ?
+			    " (read-only compatible)" : "";
+
+			(void) printf("%-37s%s\n", fi->fi_uname, ro);
+			(void) printf("     %s\n", fi->fi_desc);
+		}
+		(void) printf("\n");
+
+		(void) printf(gettext("The following legacy versions are also "
 		    "supported:\n\n"));
 		(void) printf(gettext("VER  DESCRIPTION\n"));
 		(void) printf("---  -----------------------------------------"
@@ -4618,32 +4861,44 @@ zpool_do_upgrade(int argc, char **argv)
 		(void) printf(gettext("\nFor more information on a particular "
 		    "version, including supported releases,\n"));
 		(void) printf(gettext("see the ZFS Administration Guide.\n\n"));
-	} else if (argc == 0) {
+	} else if (argc == 0 && upgradeall) {
-		int notfound;
+		cb.cb_first = B_TRUE;
-
 		ret = zpool_iter(g_zfs, upgrade_cb, &cb);
-		notfound = cb.cb_first;
+		if (ret == 0 && cb.cb_first) {
-
+			if (cb.cb_version == SPA_VERSION) {
-		if (!cb.cb_all && ret == 0) {
+				(void) printf(gettext("All pools are already "
-			if (!cb.cb_first)
+				    "formatted using feature flags.\n\n"));
-				(void) printf("\n");
+				(void) printf(gettext("Every feature flags "
-			cb.cb_first = B_TRUE;
+				    "pool already has all supported features "
-			cb.cb_newer = B_TRUE;
+				    "enabled.\n"));
-			ret = zpool_iter(g_zfs, upgrade_cb, &cb);
+			} else {
-			if (!cb.cb_first) {
+				(void) printf(gettext("All pools are already "
-				notfound = B_FALSE;
+				    "formatted with version %llu or higher.\n"),
-				(void) printf("\n");
+				    cb.cb_version);
 			}
 		}
+	} else if (argc == 0) {
+		cb.cb_first = B_TRUE;
+		ret = zpool_iter(g_zfs, upgrade_list_older_cb, &cb);
+		assert(ret == 0);
 
-		if (ret == 0) {
+		if (cb.cb_first) {
-			if (notfound)
+			(void) printf(gettext("All pools are formatted "
-				(void) printf(gettext("All pools are formatted "
+			    "using feature flags.\n\n"));
-				    "using this version.\n"));
+		} else {
-			else if (!cb.cb_all)
+			(void) printf(gettext("\nUse 'zpool upgrade -v' "
-				(void) printf(gettext("Use 'zpool upgrade -v' "
+			    "for a list of available legacy versions.\n"));
-				    "for a list of available versions and "
+		}
-				    "their associated\nfeatures.\n"));
+
+		cb.cb_first = B_TRUE;
+		ret = zpool_iter(g_zfs, upgrade_list_disabled_cb, &cb);
+		assert(ret == 0);
+
+		if (cb.cb_first) {
+			(void) printf(gettext("Every feature flags pool has "
+			    "all supported features enabled.\n"));
+		} else {
+			(void) printf(gettext("\n"));
 		}
 	} else {
 		ret = for_each_pool(argc, argv, B_FALSE, NULL,

cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h

@@ -316,7 +316,8 @@ typedef enum {
 	 * requiring administrative attention.  There is no corresponding
 	 * message ID.
 	 */
-	ZPOOL_STATUS_VERSION_OLDER,	/* older on-disk version */
+	ZPOOL_STATUS_VERSION_OLDER,	/* older legacy on-disk version */
+	ZPOOL_STATUS_FEAT_DISABLED,	/* supported features are disabled */
 	ZPOOL_STATUS_RESILVERING,	/* device being resilvered */
 	ZPOOL_STATUS_OFFLINE_DEV,	/* device online */
 	ZPOOL_STATUS_REMOVED_DEV,	/* removed device */

cddl/contrib/opensolaris/lib/libzfs/common/libzfs_status.c

@@ -44,6 +44,7 @@
 #include <string.h>
 #include <unistd.h>
 #include "libzfs_impl.h"
+#include "zfeature_common.h"
 
 /*
  * Message ID table.  This must be kept in sync with the ZPOOL_STATUS_* defines
@@ -319,6 +320,30 @@ check_status(nvlist_t *config, boolean_t isimport)
 	if (SPA_VERSION_IS_SUPPORTED(version) && version != SPA_VERSION)
 		return (ZPOOL_STATUS_VERSION_OLDER);
 
+	/*
+	 * Usable pool with disabled features
+	 */
+	if (version >= SPA_VERSION_FEATURES) {
+		int i;
+		nvlist_t *feat;
+
+		if (isimport) {
+			feat = fnvlist_lookup_nvlist(config,
+			    ZPOOL_CONFIG_LOAD_INFO);
+			feat = fnvlist_lookup_nvlist(feat,
+			    ZPOOL_CONFIG_ENABLED_FEAT);
+		} else {
+			feat = fnvlist_lookup_nvlist(config,
+			    ZPOOL_CONFIG_FEATURE_STATS);
+		}
+
+		for (i = 0; i < SPA_FEATURES; i++) {
+			zfeature_info_t *fi = &spa_feature_table[i];
+			if (!nvlist_exists(feat, fi->fi_guid))
+				return (ZPOOL_STATUS_FEAT_DISABLED);
+		}
+	}
+
 	return (ZPOOL_STATUS_OK);
 }
 

cddl/lib/libzfs/Makefile

@@ -6,8 +6,8 @@
 .PATH: ${.CURDIR}/../../../cddl/contrib/opensolaris/lib/libzfs/common
 
 LIB=	zfs
-DPADD=	${LIBMD} ${LIBPTHREAD} ${LIBUMEM} ${LIBUTIL} ${LIBM}
+DPADD=	${LIBMD} ${LIBPTHREAD} ${LIBUMEM} ${LIBUTIL} ${LIBM} ${LIBNVPAIR}
-LDADD=	-lmd -lpthread -lumem -lutil -lm
+LDADD=	-lmd -lpthread -lumem -lutil -lm -lnvpair
 
 SRCS=	deviceid.c \
 	fsshare.c \

rescue/rescue/Makefile

@@ -123,7 +123,7 @@ CRUNCH_LIBS+= -lalias -lcam -lcurses -ldevstat -lipsec
 CRUNCH_LIBS+= -lipx
 .endif
 .if ${MK_ZFS} != "no"
-CRUNCH_LIBS+= -lavl -lnvpair -lzfs -lpthread -luutil -lumem
+CRUNCH_LIBS+= -lavl -lzfs -lnvpair -lpthread -luutil -lumem
 .endif
 CRUNCH_LIBS+= -lgeom -lbsdxml -lkiconv -lmd -lsbuf -lufs -lz
 

sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c

@@ -2172,7 +2172,7 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config,
 
 	if (spa_version(spa) >= SPA_VERSION_FEATURES) {
 		boolean_t missing_feat_read = B_FALSE;
-		nvlist_t *unsup_feat;
+		nvlist_t *unsup_feat, *enabled_feat;
 
 		if (spa_dir_prop(spa, DMU_POOL_FEATURES_FOR_READ,
 		    &spa->spa_feat_for_read_obj) != 0) {
@@ -2189,27 +2189,32 @@ spa_load_impl(spa_t *spa, uint64_t pool_guid, nvlist_t *config,
 			return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
 		}
 
-		VERIFY(nvlist_alloc(&unsup_feat, NV_UNIQUE_NAME, KM_SLEEP) ==
+		enabled_feat = fnvlist_alloc();
-		    0);
+		unsup_feat = fnvlist_alloc();
 
 		if (!feature_is_supported(spa->spa_meta_objset,
 		    spa->spa_feat_for_read_obj, spa->spa_feat_desc_obj,
-		    unsup_feat))
+		    unsup_feat, enabled_feat))
 			missing_feat_read = B_TRUE;
 
 		if (spa_writeable(spa) || state == SPA_LOAD_TRYIMPORT) {
 			if (!feature_is_supported(spa->spa_meta_objset,
 			    spa->spa_feat_for_write_obj, spa->spa_feat_desc_obj,
-			    unsup_feat))
+			    unsup_feat, enabled_feat)) {
 				missing_feat_write = B_TRUE;
+			}
 		}
 
+		fnvlist_add_nvlist(spa->spa_load_info,
+		    ZPOOL_CONFIG_ENABLED_FEAT, enabled_feat);
+
 		if (!nvlist_empty(unsup_feat)) {
-			VERIFY(nvlist_add_nvlist(spa->spa_load_info,
+			fnvlist_add_nvlist(spa->spa_load_info,
-			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat) == 0);
+			    ZPOOL_CONFIG_UNSUP_FEAT, unsup_feat);
 		}
 
-		nvlist_free(unsup_feat);
+		fnvlist_free(enabled_feat);
+		fnvlist_free(unsup_feat);
 
 		if (!missing_feat_read) {
 			fnvlist_add_boolean(spa->spa_load_info,

sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfeature.h

@@ -35,7 +35,7 @@ extern "C" {
 #endif
 
 extern boolean_t feature_is_supported(objset_t *os, uint64_t obj,
-    uint64_t desc_obj, nvlist_t *unsup_feat);
+    uint64_t desc_obj, nvlist_t *unsup_feat, nvlist_t *enabled_feat);
 
 struct spa;
 extern void spa_feature_create_zap_objects(struct spa *, dmu_tx_t *);

sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfeature.c

@@ -173,7 +173,7 @@ typedef enum {
  */
 boolean_t
 feature_is_supported(objset_t *os, uint64_t obj, uint64_t desc_obj,
-    nvlist_t *unsup_feat)
+    nvlist_t *unsup_feat, nvlist_t *enabled_feat)
 {
 	boolean_t supported;
 	zap_cursor_t zc;
@@ -186,11 +186,16 @@ feature_is_supported(objset_t *os, uint64_t obj, uint64_t desc_obj,
 		ASSERT(za.za_integer_length == sizeof (uint64_t) &&
 		    za.za_num_integers == 1);
 
+		if (NULL != enabled_feat) {
+			fnvlist_add_uint64(enabled_feat, za.za_name,
+			    za.za_first_integer);
+		}
+
 		if (za.za_first_integer != 0 &&
 		    !zfeature_is_supported(za.za_name)) {
 			supported = B_FALSE;
 
-			if (unsup_feat != NULL) {
+			if (NULL != unsup_feat) {
 				char *desc = "";
 				char buf[MAXPATHLEN];
 

sys/cddl/contrib/opensolaris/uts/common/sys/fs/zfs.h

@@ -520,6 +520,7 @@ typedef struct zpool_rewind_policy {
 #define	ZPOOL_CONFIG_LOAD_INFO		"load_info"	/* not stored on disk */
 #define	ZPOOL_CONFIG_REWIND_INFO	"rewind_info"	/* not stored on disk */
 #define	ZPOOL_CONFIG_UNSUP_FEAT		"unsup_feat"	/* not stored on disk */
+#define	ZPOOL_CONFIG_ENABLED_FEAT	"enabled_feat"	/* not stored on disk */
 #define	ZPOOL_CONFIG_CAN_RDONLY		"can_rdonly"	/* not stored on disk */
 #define	ZPOOL_CONFIG_FEATURES_FOR_READ	"features_for_read"
 #define	ZPOOL_CONFIG_FEATURE_STATS	"feature_stats"	/* not stored on disk */