d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MFOpenZFS: Add support for boot environment data to be stored in the label

Browse Source 
We are building new bootonce mechanism (previously zfs bootnext) and it is
based on this OpenZFS change. Since this patch is nicely self contained,
I am commiting it as is, and we can stack our changes.

Original patch description follows:

Modern bootloaders leverage data stored in the root filesystem to
enable some of their powerful features. GRUB specifically has a grubenv
file which can store large amounts of configuration data that can be
read and written at boot time and during normal operation. This allows
sysadmins to configure useful features like automated failover after
failed boot attempts. Unfortunately, due to the Copy-on-Write nature
of ZFS, the standard behavior of these tools cannot handle writing to
ZFS files safely at boot time. We need an alternative way to store
data that allows the bootloader to make changes to the data.

This work is very similar to work that was done on Illumos to enable
similar functionality in the FreeBSD bootloader. This patch is different
in that the data being stored is a raw grubenv file; this file can store
arbitrary variables and values, and the scripting provided by grub is
powerful enough that special structures are not required to implement
advanced behavior.

We repurpose the second padding area in each label to store the grubenv
file, protected by an embedded checksum. We add two ioctls to get and
set this data, and libzfs_core and libzfs functions to access them more
easily. There are no direct command line interfaces to these functions;
these will be added directly to the bootloader utilities.

Reviewed-by: Pavel Zakharov <pavel.zakharov@delphix.com>
Reviewed-by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #10009

Obtained from:	OpenZFS
Sponsored by:	Netflix, Klara Inc.
This commit is contained in:
Toomas Soome 2020-08-05 14:32:20 +00:00
parent 491ceb65ec
commit 722c2b4aca
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=363911
11 changed files with 310 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cddl/contrib/opensolaris/cmd/zinject/translate.c
View File

@ -20,7 +20,7 @@
*/
/*
* Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012 by Delphix. All rights reserved.
* Copyright (c) 2012, 2020 by Delphix. All rights reserved.
*/
#include <libzfs.h>
@ -484,7 +484,7 @@ translate_device(const char *pool, const char *device, err_type_t label_type,
record->zi_end = record->zi_start + VDEV_PAD_SIZE - 1;
break;
case TYPE_LABEL_PAD2:
record->zi_start = offsetof(vdev_label_t, vl_pad2);
record->zi_start = offsetof(vdev_label_t, vl_be);
record->zi_end = record->zi_start + VDEV_PAD_SIZE - 1;
break;
}

2
cddl/contrib/opensolaris/lib/libzfs/common/libzfs.h
View File

@ -837,6 +837,8 @@ extern int zpool_in_use(libzfs_handle_t *, int, pool_state_t *, char **,
extern int zpool_read_label(int, nvlist_t **);
extern int zpool_read_all_labels(int, nvlist_t **);
extern int zpool_clear_label(int);
extern int zpool_set_bootenv(zpool_handle_t *, const char *);
extern int zpool_get_bootenv(zpool_handle_t *, char *, size_t, off_t);
/* is this zvol valid for use as a dump device? */
extern int zvol_check_dump_config(char *);

40
cddl/contrib/opensolaris/lib/libzfs/common/libzfs_pool.c
View File

@ -21,7 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2017 by Delphix. All rights reserved.
* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
* Copyright (c) 2013, Joyent, Inc. All rights reserved.
* Copyright 2016 Nexenta Systems, Inc.
* Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>
@ -395,7 +395,7 @@ zpool_get_prop(zpool_handle_t *zhp, zpool_prop_t prop, char *buf, size_t len,
* Assuming bootfs is a valid dataset name.
*/
static boolean_t
bootfs_name_valid(const char *pool, char *bootfs)
bootfs_name_valid(const char *pool, const char *bootfs)
{
int len = strlen(pool);
@ -4233,6 +4233,42 @@ zpool_obj_to_path(zpool_handle_t *zhp, uint64_t dsobj, uint64_t obj,
free(mntpnt);
}
int
zpool_set_bootenv(zpool_handle_t *zhp, const char *envmap)
{
int error = lzc_set_bootenv(zhp->zpool_name, envmap);
if (error != 0) {
(void) zpool_standard_error_fmt(zhp->zpool_hdl, error,
dgettext(TEXT_DOMAIN,
"error setting bootenv in pool '%s'"), zhp->zpool_name);
}
return (error);
}
int
zpool_get_bootenv(zpool_handle_t *zhp, char *outbuf, size_t size, off_t offset)
{
nvlist_t *nvl;
int error = lzc_get_bootenv(zhp->zpool_name, &nvl);;
if (error != 0) {
(void) zpool_standard_error_fmt(zhp->zpool_hdl, error,
dgettext(TEXT_DOMAIN,
"error getting bootenv in pool '%s'"), zhp->zpool_name);
return (-1);
}
char *envmap = fnvlist_lookup_string(nvl, "envmap");
if (offset >= strlen(envmap)) {
fnvlist_free(nvl);
return (0);
}
strlcpy(outbuf, envmap + offset, size);
int bytes = MIN(strlen(envmap + offset), size);
fnvlist_free(nvl);
return (bytes);
}
#ifdef illumos
/*
* Read the EFI label from the config, if a label does not exist then

24
cddl/contrib/opensolaris/lib/libzfs_core/common/libzfs_core.c
View File

@ -20,7 +20,7 @@
*/
/*
* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
* Copyright (c) 2012, 2020 by Delphix. All rights reserved.
* Copyright (c) 2013 Steven Hartland. All rights reserved.
* Copyright (c) 2014 Integros [integros.com]
* Copyright 2017 RackTop Systems.
@ -1210,3 +1210,25 @@ lzc_initialize(const char *poolname, pool_initialize_func_t cmd_type,
return (error);
}
/*
* Set the bootenv contents for the given pool.
*/
int
lzc_set_bootenv(const char *pool, const char *env)
{
nvlist_t *args = fnvlist_alloc();
fnvlist_add_string(args, "envmap", env);
int error = lzc_ioctl(ZFS_IOC_SET_BOOTENV, pool, args, NULL);
fnvlist_free(args);
return (error);
}
/*
* Get the contents of the bootenv of the given pool.
*/
int
lzc_get_bootenv(const char *pool, nvlist_t **outnvl)
{
return (lzc_ioctl(ZFS_IOC_GET_BOOTENV, pool, NULL, outnvl));
}

4
cddl/contrib/opensolaris/lib/libzfs_core/common/libzfs_core.h
View File

@ -20,7 +20,7 @@
*/
/*
* Copyright (c) 2012, 2016 by Delphix. All rights reserved.
* Copyright (c) 2012, 2020 by Delphix. All rights reserved.
* Copyright (c) 2013 by Martin Matuska <mm@FreeBSD.org>. All rights reserved.
* Copyright 2017 RackTop Systems.
* Copyright (c) 2017 Datto Inc.
@ -105,6 +105,8 @@ int lzc_channel_program_nosync(const char *, const char *, uint64_t,
int lzc_pool_checkpoint(const char *);
int lzc_pool_checkpoint_discard(const char *);
int lzc_set_bootenv(const char *, const char *);
int lzc_get_bootenv(const char *, nvlist_t **);
#ifdef __cplusplus
}
#endif

4
sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/vdev.h
View File

@ -21,7 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2017 by Delphix. All rights reserved.
* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
* Copyright (c) 2017, Intel Corporation.
*/
@ -173,6 +173,8 @@ extern nvlist_t *vdev_label_read_config(vdev_t *vd, uint64_t txg);
extern void vdev_uberblock_load(vdev_t *, struct uberblock *, nvlist_t **);
extern void vdev_label_write(zio_t *zio, vdev_t *vd, int l, abd_t *buf, uint64_t
offset, uint64_t size, zio_done_func_t *done, void *priv, int flags);
extern int vdev_label_read_bootenv(vdev_t *, nvlist_t *);
extern int vdev_label_write_bootenv(vdev_t *, char *);
typedef enum {
VDEV_LABEL_CREATE, /* create/add a new device */

26
sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/vdev_impl.h
View File

@ -20,7 +20,7 @@
*/
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2018 by Delphix. All rights reserved.
* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
* Copyright (c) 2017, Intel Corporation.
*/
@ -392,7 +392,7 @@ struct vdev {
#define VDEV_RAIDZ_MAXPARITY 3
#define VDEV_PAD_SIZE (8 << 10)
/* 2 padding areas (vl_pad1 and vl_pad2) to skip */
/* 2 padding areas (vl_pad1 and vl_be) to skip */
#define VDEV_SKIP_SIZE VDEV_PAD_SIZE * 2
#define VDEV_PHYS_SIZE (112 << 10)
#define VDEV_UBERBLOCK_RING (128 << 10)
@ -419,9 +419,29 @@ typedef struct vdev_phys {
zio_eck_t vp_zbt;
} vdev_phys_t;
typedef enum vbe_vers {
/* The bootenv file is stored as ascii text in the envblock */
VB_RAW = 0,
/*
* The bootenv file is converted to an nvlist and then packed into the
* envblock.
*/
VB_NVLIST = 1
} vbe_vers_t;
typedef struct vdev_boot_envblock {
uint64_t vbe_version;
char vbe_bootenv[VDEV_PAD_SIZE - sizeof (uint64_t) -
sizeof (zio_eck_t)];
zio_eck_t vbe_zbt;
} vdev_boot_envblock_t;
CTASSERT(sizeof (vdev_boot_envblock_t) == VDEV_PAD_SIZE);
typedef struct vdev_label {
char vl_pad1[VDEV_PAD_SIZE]; /* 8K */
char vl_pad2[VDEV_PAD_SIZE]; /* 8K */
vdev_boot_envblock_t vl_be; /* 8K */
vdev_phys_t vl_vdev_phys; /* 112K */
char vl_uberblock[VDEV_UBERBLOCK_RING]; /* 128K */
} vdev_label_t; /* 256K total */

2
sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev.c
View File

@ -1566,7 +1566,7 @@ vdev_probe(vdev_t *vd, zio_t *zio)
for (int l = 1; l < VDEV_LABELS; l++) {
zio_nowait(zio_read_phys(pio, vd,
vdev_label_offset(vd->vdev_psize, l,
offsetof(vdev_label_t, vl_pad2)), VDEV_PAD_SIZE,
offsetof(vdev_label_t, vl_be)), VDEV_PAD_SIZE,
abd_alloc_for_io(VDEV_PAD_SIZE, B_TRUE),
ZIO_CHECKSUM_OFF, vdev_probe_done, vps,
ZIO_PRIORITY_SYNC_READ, vps->vps_flags, B_TRUE));

158
sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_label.c
View File

@ -21,7 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
* Copyright (c) 2012, 2020 by Delphix. All rights reserved.
* Copyright (c) 2017, Intel Corporation.
* Copyright 2019 Joyent, Inc.
*/
@ -781,7 +781,7 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason)
nvlist_t *label;
vdev_phys_t *vp;
abd_t *vp_abd;
abd_t *pad2;
abd_t *bootenv;
uberblock_t *ub;
abd_t *ub_abd;
zio_t *zio;
@ -956,8 +956,8 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason)
ub->ub_txg = 0;
/* Initialize the 2nd padding area. */
pad2 = abd_alloc_for_io(VDEV_PAD_SIZE, B_TRUE);
abd_zero(pad2, VDEV_PAD_SIZE);
bootenv = abd_alloc_for_io(VDEV_PAD_SIZE, B_TRUE);
abd_zero(bootenv, VDEV_PAD_SIZE);
/*
* Write everything in parallel.
@ -976,8 +976,8 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason)
* Zero out the 2nd padding area where it might have
* left over data from previous filesystem format.
*/
vdev_label_write(zio, vd, l, pad2,
offsetof(vdev_label_t, vl_pad2),
vdev_label_write(zio, vd, l, bootenv,
offsetof(vdev_label_t, vl_be),
VDEV_PAD_SIZE, NULL, NULL, flags);
vdev_label_write(zio, vd, l, ub_abd,
@ -993,7 +993,7 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason)
}
nvlist_free(label);
abd_free(pad2);
abd_free(bootenv);
abd_free(ub_abd);
abd_free(vp_abd);
@ -1016,6 +1016,148 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason)
return (error);
}
/*
* Done callback for vdev_label_read_bootenv_impl. If this is the first
* callback to finish, store our abd in the callback pointer. Otherwise, we
* just free our abd and return.
*/
static void
vdev_label_read_bootenv_done(zio_t *zio)
{
zio_t *rio = zio->io_private;
abd_t **cbp = rio->io_private;
ASSERT3U(zio->io_size, ==, VDEV_PAD_SIZE);
if (zio->io_error == 0) {
mutex_enter(&rio->io_lock);
if (*cbp == NULL) {
/* Will free this buffer in vdev_label_read_bootenv. */
*cbp = zio->io_abd;
} else {
abd_free(zio->io_abd);
}
mutex_exit(&rio->io_lock);
} else {
abd_free(zio->io_abd);
}
}
static void
vdev_label_read_bootenv_impl(zio_t *zio, vdev_t *vd, int flags)
{
for (int c = 0; c < vd->vdev_children; c++)
vdev_label_read_bootenv_impl(zio, vd->vdev_child[c], flags);
/*
* We just use the first label that has a correct checksum; the
* bootloader should have rewritten them all to be the same on boot,
* and any changes we made since boot have been the same across all
* labels.
*
* While grub supports writing to all four labels, other bootloaders
* don't, so we only use the first two labels to store boot
* information.
*/
if (vd->vdev_ops->vdev_op_leaf && vdev_readable(vd)) {
for (int l = 0; l < VDEV_LABELS / 2; l++) {
vdev_label_read(zio, vd, l,
abd_alloc_linear(VDEV_PAD_SIZE, B_FALSE),
offsetof(vdev_label_t, vl_be), VDEV_PAD_SIZE,
vdev_label_read_bootenv_done, zio, flags);
}
}
}
int
vdev_label_read_bootenv(vdev_t *rvd, nvlist_t *command)
{
spa_t *spa = rvd->vdev_spa;
abd_t *abd = NULL;
int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL |
ZIO_FLAG_SPECULATIVE | ZIO_FLAG_TRYHARD;
ASSERT(command);
ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
zio_t *zio = zio_root(spa, NULL, &abd, flags);
vdev_label_read_bootenv_impl(zio, rvd, flags);
int err = zio_wait(zio);
if (abd != NULL) {
vdev_boot_envblock_t *vbe = abd_to_buf(abd);
if (vbe->vbe_version != VB_RAW) {
abd_free(abd);
return (SET_ERROR(ENOTSUP));
}
vbe->vbe_bootenv[sizeof (vbe->vbe_bootenv) - 1] = '\0';
fnvlist_add_string(command, "envmap", vbe->vbe_bootenv);
/* abd was allocated in vdev_label_read_bootenv_impl() */
abd_free(abd);
/* If we managed to read any successfully, return success. */
return (0);
}
return (err);
}
int
vdev_label_write_bootenv(vdev_t *vd, char *envmap)
{
zio_t *zio;
spa_t *spa = vd->vdev_spa;
vdev_boot_envblock_t *bootenv;
int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL;
int error = ENXIO;
if (strlen(envmap) >= sizeof (bootenv->vbe_bootenv)) {
return (SET_ERROR(E2BIG));
}
ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
for (int c = 0; c < vd->vdev_children; c++) {
int child_err = vdev_label_write_bootenv(vd->vdev_child[c],
envmap);
/*
* As long as any of the disks managed to write all of their
* labels successfully, return success.
*/
if (child_err == 0)
error = child_err;
}
if (!vd->vdev_ops->vdev_op_leaf || vdev_is_dead(vd) ||
!vdev_writeable(vd)) {
return (error);
}
ASSERT3U(sizeof (*bootenv), ==, VDEV_PAD_SIZE);
abd_t *abd = abd_alloc_for_io(VDEV_PAD_SIZE, B_TRUE);
abd_zero(abd, VDEV_PAD_SIZE);
bootenv = abd_borrow_buf_copy(abd, VDEV_PAD_SIZE);
char *buf = bootenv->vbe_bootenv;
(void) strlcpy(buf, envmap, sizeof (bootenv->vbe_bootenv));
bootenv->vbe_version = VB_RAW;
abd_return_buf_copy(abd, bootenv, VDEV_PAD_SIZE);
retry:
zio = zio_root(spa, NULL, NULL, flags);
for (int l = 0; l < VDEV_LABELS / 2; l++) {
vdev_label_write(zio, vd, l, abd,
offsetof(vdev_label_t, vl_be),
VDEV_PAD_SIZE, NULL, NULL, flags);
}
error = zio_wait(zio);
if (error != 0 && !(flags & ZIO_FLAG_TRYHARD)) {
flags |= ZIO_FLAG_TRYHARD;
goto retry;
}
abd_free(abd);
return (error);
}
int
vdev_label_write_pad2(vdev_t *vd, const char *buf, size_t size)
{
@ -1042,7 +1184,7 @@ vdev_label_write_pad2(vdev_t *vd, const char *buf, size_t size)
retry:
zio = zio_root(spa, NULL, NULL, flags);
vdev_label_write(zio, vd, 0, pad2,
offsetof(vdev_label_t, vl_pad2),
offsetof(vdev_label_t, vl_be),
VDEV_PAD_SIZE, NULL, NULL, flags);
error = zio_wait(zio);
if (error != 0 && !(flags & ZIO_FLAG_TRYHARD)) {

62
sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
View File

@ -3654,6 +3654,58 @@ zfs_ioc_log_history(const char *unused, nvlist_t *innvl, nvlist_t *outnvl)
return (error);
}
/*
* This ioctl is used to set the bootenv configuration on the current
* pool. This configuration is stored in the second padding area of the label,
* and it is used by the GRUB bootloader used on Linux to store the contents
* of the grubenv file. The file is stored as raw ASCII, and is protected by
* an embedded checksum. By default, GRUB will check if the boot filesystem
* supports storing the environment data in a special location, and if so,
* will invoke filesystem specific logic to retrieve it. This can be overriden
* by a variable, should the user so desire.
*/
/* ARGSUSED */
static const zfs_ioc_key_t zfs_keys_set_bootenv[] = {
{"envmap", DATA_TYPE_STRING, 0},
};
static int
zfs_ioc_set_bootenv(const char *name, nvlist_t *innvl, nvlist_t *outnvl)
{
char *envmap;
int error;
spa_t *spa;
envmap = fnvlist_lookup_string(innvl, "envmap");
if ((error = spa_open(name, &spa, FTAG)) != 0)
return (error);
spa_vdev_state_enter(spa, SCL_ALL);
error = vdev_label_write_bootenv(spa->spa_root_vdev, envmap);
(void) spa_vdev_state_exit(spa, NULL, 0);
spa_close(spa, FTAG);
return (error);
}
static const zfs_ioc_key_t zfs_keys_get_bootenv[] = {
/* no nvl keys */
};
/* ARGSUSED */
static int
zfs_ioc_get_bootenv(const char *name, nvlist_t *innvl, nvlist_t *outnvl)
{
spa_t *spa;
int error;
if ((error = spa_open(name, &spa, FTAG)) != 0)
return (error);
spa_vdev_state_enter(spa, SCL_ALL);
error = vdev_label_read_bootenv(spa->spa_root_vdev, outnvl);
(void) spa_vdev_state_exit(spa, NULL, 0);
spa_close(spa, FTAG);
return (error);
}
#ifdef __FreeBSD__
static const zfs_ioc_key_t zfs_keys_nextboot[] = {
{"command", DATA_TYPE_STRING, 0},
@ -6567,6 +6619,16 @@ zfs_ioctl_init(void)
zfs_secpolicy_config, POOL_NAME, POOL_CHECK_SUSPENDED, B_TRUE,
B_TRUE, zfs_keys_pool_reopen, ARRAY_SIZE(zfs_keys_pool_reopen));
zfs_ioctl_register("set_bootenv", ZFS_IOC_SET_BOOTENV,
zfs_ioc_set_bootenv, zfs_secpolicy_config, POOL_NAME,
POOL_CHECK_SUSPENDED | POOL_CHECK_READONLY, B_FALSE, B_TRUE,
zfs_keys_set_bootenv, ARRAY_SIZE(zfs_keys_set_bootenv));
zfs_ioctl_register("get_bootenv", ZFS_IOC_GET_BOOTENV,
zfs_ioc_get_bootenv, zfs_secpolicy_none, POOL_NAME,
POOL_CHECK_SUSPENDED, B_FALSE, B_TRUE,
zfs_keys_get_bootenv, ARRAY_SIZE(zfs_keys_get_bootenv));
/* IOCTLS that use the legacy function signature */
zfs_ioctl_register_legacy(ZFS_IOC_POOL_FREEZE, zfs_ioc_pool_freeze,

4
sys/cddl/contrib/opensolaris/uts/common/sys/fs/zfs.h
View File

@ -21,7 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2018 by Delphix. All rights reserved.
* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
* Copyright (c) 2012, Martin Matuska <mm@FreeBSD.org>. All rights reserved.
* Copyright (c) 2014 Integros [integros.com]
@ -1057,6 +1057,8 @@ typedef enum zfs_ioc {
ZFS_IOC_POOL_DISCARD_CHECKPOINT,
ZFS_IOC_POOL_INITIALIZE,
ZFS_IOC_POOL_SYNC,
ZFS_IOC_SET_BOOTENV,
ZFS_IOC_GET_BOOTENV,
ZFS_IOC_LAST
} zfs_ioc_t;