2013-09-04 12:00:57 +00:00
|
|
|
/*
|
|
|
|
* CDDL HEADER START
|
|
|
|
*
|
|
|
|
* The contents of this file are subject to the terms of the
|
|
|
|
* Common Development and Distribution License (the "License").
|
|
|
|
* You may not use this file except in compliance with the License.
|
|
|
|
*
|
|
|
|
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
|
|
|
* or http://www.opensolaris.org/os/licensing.
|
|
|
|
* See the License for the specific language governing permissions
|
|
|
|
* and limitations under the License.
|
|
|
|
*
|
|
|
|
* When distributing Covered Code, include this CDDL HEADER in each
|
|
|
|
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
|
|
|
* If applicable, add the following below this CDDL HEADER, with the
|
|
|
|
* fields enclosed by brackets "[]" replaced with your own identifying
|
|
|
|
* information: Portions Copyright [yyyy] [name of copyright owner]
|
|
|
|
*
|
|
|
|
* CDDL HEADER END
|
|
|
|
*/
|
|
|
|
/*
|
|
|
|
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
|
2018-10-16 18:15:04 +00:00
|
|
|
* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
|
2013-05-25 02:06:23 +00:00
|
|
|
* Copyright (c) 2013 Steven Hartland. All rights reserved.
|
2015-04-01 13:07:48 +00:00
|
|
|
* Copyright (c) 2013 by Joyent, Inc. All rights reserved.
|
2014-03-22 09:07:14 +00:00
|
|
|
* Copyright (c) 2016 Actifio, Inc. All rights reserved.
|
2013-09-04 12:00:57 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <sys/zfs_context.h>
|
|
|
|
#include <sys/dsl_userhold.h>
|
|
|
|
#include <sys/dsl_dataset.h>
|
|
|
|
#include <sys/dsl_synctask.h>
|
2018-02-08 16:16:23 +00:00
|
|
|
#include <sys/dsl_destroy.h>
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
#include <sys/dsl_bookmark.h>
|
2013-09-04 12:00:57 +00:00
|
|
|
#include <sys/dmu_tx.h>
|
|
|
|
#include <sys/dsl_pool.h>
|
|
|
|
#include <sys/dsl_dir.h>
|
|
|
|
#include <sys/dmu_traverse.h>
|
|
|
|
#include <sys/dsl_scan.h>
|
|
|
|
#include <sys/dmu_objset.h>
|
|
|
|
#include <sys/zap.h>
|
|
|
|
#include <sys/zfeature.h>
|
|
|
|
#include <sys/zfs_ioctl.h>
|
|
|
|
#include <sys/dsl_deleg.h>
|
2013-10-08 17:13:05 +00:00
|
|
|
#include <sys/dmu_impl.h>
|
2014-03-22 09:07:14 +00:00
|
|
|
#include <sys/zvol.h>
|
2018-02-08 16:16:23 +00:00
|
|
|
#include <sys/zcp.h>
|
2019-07-26 17:54:14 +00:00
|
|
|
#include <sys/dsl_deadlist.h>
|
|
|
|
#include <sys/zthr.h>
|
|
|
|
#include <sys/spa_impl.h>
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2013-07-29 18:55:16 +00:00
|
|
|
int
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_destroy_snapshot_check_impl(dsl_dataset_t *ds, boolean_t defer)
|
|
|
|
{
|
2015-04-02 03:44:32 +00:00
|
|
|
if (!ds->ds_is_snapshot)
|
2013-03-08 18:41:28 +00:00
|
|
|
return (SET_ERROR(EINVAL));
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
if (dsl_dataset_long_held(ds))
|
2013-03-08 18:41:28 +00:00
|
|
|
return (SET_ERROR(EBUSY));
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Only allow deferred destroy on pools that support it.
|
|
|
|
* NOTE: deferred destroy is only supported on snapshots.
|
|
|
|
*/
|
|
|
|
if (defer) {
|
|
|
|
if (spa_version(ds->ds_dir->dd_pool->dp_spa) <
|
|
|
|
SPA_VERSION_USERREFS)
|
2013-03-08 18:41:28 +00:00
|
|
|
return (SET_ERROR(ENOTSUP));
|
2013-09-04 12:00:57 +00:00
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If this snapshot has an elevated user reference count,
|
|
|
|
* we can't destroy it yet.
|
|
|
|
*/
|
|
|
|
if (ds->ds_userrefs > 0)
|
2013-03-08 18:41:28 +00:00
|
|
|
return (SET_ERROR(EBUSY));
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Can't delete a branch point.
|
|
|
|
*/
|
2015-04-01 15:14:34 +00:00
|
|
|
if (dsl_dataset_phys(ds)->ds_num_children > 1)
|
2013-03-08 18:41:28 +00:00
|
|
|
return (SET_ERROR(EEXIST));
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2018-02-08 16:16:23 +00:00
|
|
|
int
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_destroy_snapshot_check(void *arg, dmu_tx_t *tx)
|
|
|
|
{
|
2018-02-08 16:16:23 +00:00
|
|
|
dsl_destroy_snapshot_arg_t *ddsa = arg;
|
|
|
|
const char *dsname = ddsa->ddsa_name;
|
|
|
|
boolean_t defer = ddsa->ddsa_defer;
|
|
|
|
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_pool_t *dp = dmu_tx_pool(tx);
|
|
|
|
int error = 0;
|
2018-02-08 16:16:23 +00:00
|
|
|
dsl_dataset_t *ds;
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2018-02-08 16:16:23 +00:00
|
|
|
error = dsl_dataset_hold(dp, dsname, FTAG, &ds);
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2018-02-08 16:16:23 +00:00
|
|
|
/*
|
|
|
|
* If the snapshot does not exist, silently ignore it, and
|
|
|
|
* dsl_destroy_snapshot_sync() will be a no-op
|
|
|
|
* (it's "already destroyed").
|
|
|
|
*/
|
|
|
|
if (error == ENOENT)
|
|
|
|
return (0);
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2018-02-08 16:16:23 +00:00
|
|
|
if (error == 0) {
|
|
|
|
error = dsl_destroy_snapshot_check_impl(ds, defer);
|
|
|
|
dsl_dataset_rele(ds, FTAG);
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
|
2018-02-08 16:16:23 +00:00
|
|
|
return (error);
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
struct process_old_arg {
|
|
|
|
dsl_dataset_t *ds;
|
|
|
|
dsl_dataset_t *ds_prev;
|
|
|
|
boolean_t after_branch_point;
|
|
|
|
zio_t *pio;
|
|
|
|
uint64_t used, comp, uncomp;
|
|
|
|
};
|
|
|
|
|
|
|
|
static int
|
2019-07-26 17:54:14 +00:00
|
|
|
process_old_cb(void *arg, const blkptr_t *bp, boolean_t bp_freed, dmu_tx_t *tx)
|
2013-09-04 12:00:57 +00:00
|
|
|
{
|
|
|
|
struct process_old_arg *poa = arg;
|
|
|
|
dsl_pool_t *dp = poa->ds->ds_dir->dd_pool;
|
|
|
|
|
2013-12-09 18:37:51 +00:00
|
|
|
ASSERT(!BP_IS_HOLE(bp));
|
|
|
|
|
2015-04-01 15:14:34 +00:00
|
|
|
if (bp->blk_birth <= dsl_dataset_phys(poa->ds)->ds_prev_snap_txg) {
|
2019-07-26 17:54:14 +00:00
|
|
|
dsl_deadlist_insert(&poa->ds->ds_deadlist, bp, bp_freed, tx);
|
2013-09-04 12:00:57 +00:00
|
|
|
if (poa->ds_prev && !poa->after_branch_point &&
|
|
|
|
bp->blk_birth >
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(poa->ds_prev)->ds_prev_snap_txg) {
|
|
|
|
dsl_dataset_phys(poa->ds_prev)->ds_unique_bytes +=
|
2013-09-04 12:00:57 +00:00
|
|
|
bp_get_dsize_sync(dp->dp_spa, bp);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
poa->used += bp_get_dsize_sync(dp->dp_spa, bp);
|
|
|
|
poa->comp += BP_GET_PSIZE(bp);
|
|
|
|
poa->uncomp += BP_GET_UCSIZE(bp);
|
|
|
|
dsl_free_sync(poa->pio, dp, tx->tx_txg, bp);
|
|
|
|
}
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
process_old_deadlist(dsl_dataset_t *ds, dsl_dataset_t *ds_prev,
|
|
|
|
dsl_dataset_t *ds_next, boolean_t after_branch_point, dmu_tx_t *tx)
|
|
|
|
{
|
|
|
|
struct process_old_arg poa = { 0 };
|
|
|
|
dsl_pool_t *dp = ds->ds_dir->dd_pool;
|
|
|
|
objset_t *mos = dp->dp_meta_objset;
|
|
|
|
uint64_t deadlist_obj;
|
|
|
|
|
|
|
|
ASSERT(ds->ds_deadlist.dl_oldfmt);
|
|
|
|
ASSERT(ds_next->ds_deadlist.dl_oldfmt);
|
|
|
|
|
|
|
|
poa.ds = ds;
|
|
|
|
poa.ds_prev = ds_prev;
|
|
|
|
poa.after_branch_point = after_branch_point;
|
|
|
|
poa.pio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
|
|
|
|
VERIFY0(bpobj_iterate(&ds_next->ds_deadlist.dl_bpobj,
|
|
|
|
process_old_cb, &poa, tx));
|
|
|
|
VERIFY0(zio_wait(poa.pio));
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT3U(poa.used, ==, dsl_dataset_phys(ds)->ds_unique_bytes);
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
/* change snapused */
|
|
|
|
dsl_dir_diduse_space(ds->ds_dir, DD_USED_SNAP,
|
|
|
|
-poa.used, -poa.comp, -poa.uncomp, tx);
|
|
|
|
|
|
|
|
/* swap next's deadlist to our deadlist */
|
|
|
|
dsl_deadlist_close(&ds->ds_deadlist);
|
|
|
|
dsl_deadlist_close(&ds_next->ds_deadlist);
|
2015-04-01 15:14:34 +00:00
|
|
|
deadlist_obj = dsl_dataset_phys(ds)->ds_deadlist_obj;
|
|
|
|
dsl_dataset_phys(ds)->ds_deadlist_obj =
|
|
|
|
dsl_dataset_phys(ds_next)->ds_deadlist_obj;
|
|
|
|
dsl_dataset_phys(ds_next)->ds_deadlist_obj = deadlist_obj;
|
|
|
|
dsl_deadlist_open(&ds->ds_deadlist, mos,
|
|
|
|
dsl_dataset_phys(ds)->ds_deadlist_obj);
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_deadlist_open(&ds_next->ds_deadlist, mos,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds_next)->ds_deadlist_obj);
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
typedef struct remaining_clones_key {
|
|
|
|
dsl_dataset_t *rck_clone;
|
|
|
|
list_node_t rck_node;
|
|
|
|
} remaining_clones_key_t;
|
|
|
|
|
|
|
|
static remaining_clones_key_t *
|
|
|
|
rck_alloc(dsl_dataset_t *clone)
|
|
|
|
{
|
|
|
|
remaining_clones_key_t *rck = kmem_alloc(sizeof (*rck), KM_SLEEP);
|
|
|
|
rck->rck_clone = clone;
|
|
|
|
return (rck);
|
|
|
|
}
|
2018-08-22 18:03:31 +00:00
|
|
|
|
2013-09-04 12:00:57 +00:00
|
|
|
static void
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
dsl_dir_remove_clones_key_impl(dsl_dir_t *dd, uint64_t mintxg, dmu_tx_t *tx,
|
|
|
|
list_t *stack, void *tag)
|
2013-09-04 12:00:57 +00:00
|
|
|
{
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
objset_t *mos = dd->dd_pool->dp_meta_objset;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If it is the old version, dd_clones doesn't exist so we can't
|
|
|
|
* find the clones, but dsl_deadlist_remove_key() is a no-op so it
|
|
|
|
* doesn't matter.
|
|
|
|
*/
|
|
|
|
if (dsl_dir_phys(dd)->dd_clones == 0)
|
|
|
|
return;
|
2013-09-04 12:00:57 +00:00
|
|
|
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
zap_cursor_t *zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP);
|
|
|
|
zap_attribute_t *za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);
|
2018-08-22 18:03:31 +00:00
|
|
|
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
for (zap_cursor_init(zc, mos, dsl_dir_phys(dd)->dd_clones);
|
|
|
|
zap_cursor_retrieve(zc, za) == 0;
|
|
|
|
zap_cursor_advance(zc)) {
|
|
|
|
dsl_dataset_t *clone;
|
2018-08-22 18:03:31 +00:00
|
|
|
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
VERIFY0(dsl_dataset_hold_obj(dd->dd_pool,
|
|
|
|
za->za_first_integer, tag, &clone));
|
|
|
|
|
|
|
|
if (clone->ds_dir->dd_origin_txg > mintxg) {
|
|
|
|
dsl_deadlist_remove_key(&clone->ds_deadlist,
|
|
|
|
mintxg, tx);
|
|
|
|
|
|
|
|
if (dsl_dataset_remap_deadlist_exists(clone)) {
|
|
|
|
dsl_deadlist_remove_key(
|
|
|
|
&clone->ds_remap_deadlist, mintxg, tx);
|
OpenZFS 7614, 9064 - zfs device evacuation/removal
OpenZFS 7614 - zfs device evacuation/removal
OpenZFS 9064 - remove_mirror should wait for device removal to complete
This project allows top-level vdevs to be removed from the storage pool
with "zpool remove", reducing the total amount of storage in the pool.
This operation copies all allocated regions of the device to be removed
onto other devices, recording the mapping from old to new location.
After the removal is complete, read and free operations to the removed
(now "indirect") vdev must be remapped and performed at the new location
on disk. The indirect mapping table is kept in memory whenever the pool
is loaded, so there is minimal performance overhead when doing operations
on the indirect vdev.
The size of the in-memory mapping table will be reduced when its entries
become "obsolete" because they are no longer used by any block pointers
in the pool. An entry becomes obsolete when all the blocks that use
it are freed. An entry can also become obsolete when all the snapshots
that reference it are deleted, and the block pointers that reference it
have been "remapped" in all filesystems/zvols (and clones). Whenever an
indirect block is written, all the block pointers in it will be "remapped"
to their new (concrete) locations if possible. This process can be
accelerated by using the "zfs remap" command to proactively rewrite all
indirect blocks that reference indirect (removed) vdevs.
Note that when a device is removed, we do not verify the checksum of
the data that is copied. This makes the process much faster, but if it
were used on redundant vdevs (i.e. mirror or raidz vdevs), it would be
possible to copy the wrong data, when we have the correct data on e.g.
the other side of the mirror.
At the moment, only mirrors and simple top-level vdevs can be removed
and no removal is allowed if any of the top-level vdevs are raidz.
Porting Notes:
* Avoid zero-sized kmem_alloc() in vdev_compact_children().
The device evacuation code adds a dependency that
vdev_compact_children() be able to properly empty the vdev_child
array by setting it to NULL and zeroing vdev_children. Under Linux,
kmem_alloc() and related functions return a sentinel pointer rather
than NULL for zero-sized allocations.
* Remove comment regarding "mpt" driver where zfs_remove_max_segment
is initialized to SPA_MAXBLOCKSIZE.
Change zfs_condense_indirect_commit_entry_delay_ticks to
zfs_condense_indirect_commit_entry_delay_ms for consistency with
most other tunables in which delays are specified in ms.
* ZTS changes:
Use set_tunable rather than mdb
Use zpool sync as appropriate
Use sync_pool instead of sync
Kill jobs during test_removal_with_operation to allow unmount/export
Don't add non-disk names such as "mirror" or "raidz" to $DISKS
Use $TEST_BASE_DIR instead of /tmp
Increase HZ from 100 to 1000 which is more common on Linux
removal_multiple_indirection.ksh
Reduce iterations in order to not time out on the code
coverage builders.
removal_resume_export:
Functionally, the test case is correct but there exists a race
where the kernel thread hasn't been fully started yet and is
not visible. Wait for up to 1 second for the removal thread
to be started before giving up on it. Also, increase the
amount of data copied in order that the removal not finish
before the export has a chance to fail.
* MMP compatibility, the concept of concrete versus non-concrete devices
has slightly changed the semantics of vdev_writeable(). Update
mmp_random_leaf_impl() accordingly.
* Updated dbuf_remap() to handle the org.zfsonlinux:large_dnode pool
feature which is not supported by OpenZFS.
* Added support for new vdev removal tracepoints.
* Test cases removal_with_zdb and removal_condense_export have been
intentionally disabled. When run manually they pass as intended,
but when running in the automated test environment they produce
unreliable results on the latest Fedora release.
They may work better once the upstream pool import refectoring is
merged into ZoL at which point they will be re-enabled.
Authored by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Alex Reece <alex@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Richard Laager <rlaager@wiktel.com>
Reviewed by: Tim Chase <tim@chase2k.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Approved by: Garrett D'Amore <garrett@damore.org>
Ported-by: Tim Chase <tim@chase2k.com>
Signed-off-by: Tim Chase <tim@chase2k.com>
OpenZFS-issue: https://www.illumos.org/issues/7614
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/f539f1eb
Closes #6900
2016-09-22 16:30:13 +00:00
|
|
|
}
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
|
|
|
|
list_insert_head(stack, rck_alloc(clone));
|
|
|
|
} else {
|
|
|
|
dsl_dataset_rele(clone, tag);
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
}
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
zap_cursor_fini(zc);
|
|
|
|
|
|
|
|
kmem_free(za, sizeof (zap_attribute_t));
|
|
|
|
kmem_free(zc, sizeof (zap_cursor_t));
|
|
|
|
}
|
2013-09-25 22:14:47 +00:00
|
|
|
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
void
|
|
|
|
dsl_dir_remove_clones_key(dsl_dir_t *top_dd, uint64_t mintxg, dmu_tx_t *tx)
|
|
|
|
{
|
|
|
|
list_t stack;
|
|
|
|
|
|
|
|
list_create(&stack, sizeof (remaining_clones_key_t),
|
|
|
|
offsetof(remaining_clones_key_t, rck_node));
|
|
|
|
|
|
|
|
dsl_dir_remove_clones_key_impl(top_dd, mintxg, tx, &stack, FTAG);
|
|
|
|
for (remaining_clones_key_t *rck = list_remove_head(&stack);
|
|
|
|
rck != NULL; rck = list_remove_head(&stack)) {
|
|
|
|
dsl_dataset_t *clone = rck->rck_clone;
|
|
|
|
dsl_dir_t *clone_dir = clone->ds_dir;
|
|
|
|
|
|
|
|
kmem_free(rck, sizeof (*rck));
|
|
|
|
|
|
|
|
dsl_dir_remove_clones_key_impl(clone_dir, mintxg, tx,
|
|
|
|
&stack, FTAG);
|
|
|
|
dsl_dataset_rele(clone, FTAG);
|
2018-08-22 18:03:31 +00:00
|
|
|
}
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
|
|
|
|
list_destroy(&stack);
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
|
OpenZFS 7614, 9064 - zfs device evacuation/removal
OpenZFS 7614 - zfs device evacuation/removal
OpenZFS 9064 - remove_mirror should wait for device removal to complete
This project allows top-level vdevs to be removed from the storage pool
with "zpool remove", reducing the total amount of storage in the pool.
This operation copies all allocated regions of the device to be removed
onto other devices, recording the mapping from old to new location.
After the removal is complete, read and free operations to the removed
(now "indirect") vdev must be remapped and performed at the new location
on disk. The indirect mapping table is kept in memory whenever the pool
is loaded, so there is minimal performance overhead when doing operations
on the indirect vdev.
The size of the in-memory mapping table will be reduced when its entries
become "obsolete" because they are no longer used by any block pointers
in the pool. An entry becomes obsolete when all the blocks that use
it are freed. An entry can also become obsolete when all the snapshots
that reference it are deleted, and the block pointers that reference it
have been "remapped" in all filesystems/zvols (and clones). Whenever an
indirect block is written, all the block pointers in it will be "remapped"
to their new (concrete) locations if possible. This process can be
accelerated by using the "zfs remap" command to proactively rewrite all
indirect blocks that reference indirect (removed) vdevs.
Note that when a device is removed, we do not verify the checksum of
the data that is copied. This makes the process much faster, but if it
were used on redundant vdevs (i.e. mirror or raidz vdevs), it would be
possible to copy the wrong data, when we have the correct data on e.g.
the other side of the mirror.
At the moment, only mirrors and simple top-level vdevs can be removed
and no removal is allowed if any of the top-level vdevs are raidz.
Porting Notes:
* Avoid zero-sized kmem_alloc() in vdev_compact_children().
The device evacuation code adds a dependency that
vdev_compact_children() be able to properly empty the vdev_child
array by setting it to NULL and zeroing vdev_children. Under Linux,
kmem_alloc() and related functions return a sentinel pointer rather
than NULL for zero-sized allocations.
* Remove comment regarding "mpt" driver where zfs_remove_max_segment
is initialized to SPA_MAXBLOCKSIZE.
Change zfs_condense_indirect_commit_entry_delay_ticks to
zfs_condense_indirect_commit_entry_delay_ms for consistency with
most other tunables in which delays are specified in ms.
* ZTS changes:
Use set_tunable rather than mdb
Use zpool sync as appropriate
Use sync_pool instead of sync
Kill jobs during test_removal_with_operation to allow unmount/export
Don't add non-disk names such as "mirror" or "raidz" to $DISKS
Use $TEST_BASE_DIR instead of /tmp
Increase HZ from 100 to 1000 which is more common on Linux
removal_multiple_indirection.ksh
Reduce iterations in order to not time out on the code
coverage builders.
removal_resume_export:
Functionally, the test case is correct but there exists a race
where the kernel thread hasn't been fully started yet and is
not visible. Wait for up to 1 second for the removal thread
to be started before giving up on it. Also, increase the
amount of data copied in order that the removal not finish
before the export has a chance to fail.
* MMP compatibility, the concept of concrete versus non-concrete devices
has slightly changed the semantics of vdev_writeable(). Update
mmp_random_leaf_impl() accordingly.
* Updated dbuf_remap() to handle the org.zfsonlinux:large_dnode pool
feature which is not supported by OpenZFS.
* Added support for new vdev removal tracepoints.
* Test cases removal_with_zdb and removal_condense_export have been
intentionally disabled. When run manually they pass as intended,
but when running in the automated test environment they produce
unreliable results on the latest Fedora release.
They may work better once the upstream pool import refectoring is
merged into ZoL at which point they will be re-enabled.
Authored by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Alex Reece <alex@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Richard Laager <rlaager@wiktel.com>
Reviewed by: Tim Chase <tim@chase2k.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Approved by: Garrett D'Amore <garrett@damore.org>
Ported-by: Tim Chase <tim@chase2k.com>
Signed-off-by: Tim Chase <tim@chase2k.com>
OpenZFS-issue: https://www.illumos.org/issues/7614
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/f539f1eb
Closes #6900
2016-09-22 16:30:13 +00:00
|
|
|
static void
|
|
|
|
dsl_destroy_snapshot_handle_remaps(dsl_dataset_t *ds, dsl_dataset_t *ds_next,
|
|
|
|
dmu_tx_t *tx)
|
|
|
|
{
|
|
|
|
dsl_pool_t *dp = ds->ds_dir->dd_pool;
|
|
|
|
|
|
|
|
/* Move blocks to be obsoleted to pool's obsolete list. */
|
|
|
|
if (dsl_dataset_remap_deadlist_exists(ds_next)) {
|
|
|
|
if (!bpobj_is_open(&dp->dp_obsolete_bpobj))
|
|
|
|
dsl_pool_create_obsolete_bpobj(dp, tx);
|
|
|
|
|
|
|
|
dsl_deadlist_move_bpobj(&ds_next->ds_remap_deadlist,
|
|
|
|
&dp->dp_obsolete_bpobj,
|
|
|
|
dsl_dataset_phys(ds)->ds_prev_snap_txg, tx);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Merge our deadlist into next's and free it. */
|
|
|
|
if (dsl_dataset_remap_deadlist_exists(ds)) {
|
|
|
|
uint64_t remap_deadlist_object =
|
|
|
|
dsl_dataset_get_remap_deadlist_object(ds);
|
|
|
|
ASSERT(remap_deadlist_object != 0);
|
|
|
|
|
|
|
|
mutex_enter(&ds_next->ds_remap_deadlist_lock);
|
|
|
|
if (!dsl_dataset_remap_deadlist_exists(ds_next))
|
|
|
|
dsl_dataset_create_remap_deadlist(ds_next, tx);
|
|
|
|
mutex_exit(&ds_next->ds_remap_deadlist_lock);
|
|
|
|
|
|
|
|
dsl_deadlist_merge(&ds_next->ds_remap_deadlist,
|
|
|
|
remap_deadlist_object, tx);
|
|
|
|
dsl_dataset_destroy_remap_deadlist(ds, tx);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2013-09-04 12:00:57 +00:00
|
|
|
void
|
|
|
|
dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
|
|
|
|
{
|
|
|
|
int after_branch_point = FALSE;
|
|
|
|
dsl_pool_t *dp = ds->ds_dir->dd_pool;
|
|
|
|
objset_t *mos = dp->dp_meta_objset;
|
|
|
|
dsl_dataset_t *ds_prev = NULL;
|
2017-11-04 20:25:13 +00:00
|
|
|
uint64_t obj;
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
ASSERT(RRW_WRITE_HELD(&dp->dp_config_rwlock));
|
2017-01-27 19:43:42 +00:00
|
|
|
rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT3U(dsl_dataset_phys(ds)->ds_bp.blk_birth, <=, tx->tx_txg);
|
2017-01-27 19:43:42 +00:00
|
|
|
rrw_exit(&ds->ds_bp_rwlock, FTAG);
|
2018-10-01 17:42:05 +00:00
|
|
|
ASSERT(zfs_refcount_is_zero(&ds->ds_longholds));
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
if (defer &&
|
2015-04-01 15:14:34 +00:00
|
|
|
(ds->ds_userrefs > 0 ||
|
|
|
|
dsl_dataset_phys(ds)->ds_num_children > 1)) {
|
2013-09-04 12:00:57 +00:00
|
|
|
ASSERT(spa_version(dp->dp_spa) >= SPA_VERSION_USERREFS);
|
|
|
|
dmu_buf_will_dirty(ds->ds_dbuf, tx);
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_DEFER_DESTROY;
|
2019-09-12 20:28:26 +00:00
|
|
|
spa_history_log_internal_ds(ds, "defer_destroy", tx, " ");
|
2013-09-04 12:00:57 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT3U(dsl_dataset_phys(ds)->ds_num_children, <=, 1);
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
/* We need to log before removing it from the namespace. */
|
2019-09-12 20:28:26 +00:00
|
|
|
spa_history_log_internal_ds(ds, "destroy", tx, " ");
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
dsl_scan_ds_destroyed(ds, tx);
|
|
|
|
|
|
|
|
obj = ds->ds_object;
|
|
|
|
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
boolean_t book_exists = dsl_bookmark_ds_destroyed(ds, tx);
|
|
|
|
|
2017-11-04 20:25:13 +00:00
|
|
|
for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
|
2018-10-16 18:15:04 +00:00
|
|
|
if (dsl_dataset_feature_is_active(ds, f))
|
|
|
|
dsl_dataset_deactivate_feature(ds, f, tx);
|
2014-11-03 20:15:08 +00:00
|
|
|
}
|
2015-04-01 15:14:34 +00:00
|
|
|
if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
|
2013-09-04 12:00:57 +00:00
|
|
|
ASSERT3P(ds->ds_prev, ==, NULL);
|
|
|
|
VERIFY0(dsl_dataset_hold_obj(dp,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &ds_prev));
|
2013-09-04 12:00:57 +00:00
|
|
|
after_branch_point =
|
2015-04-01 15:14:34 +00:00
|
|
|
(dsl_dataset_phys(ds_prev)->ds_next_snap_obj != obj);
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
dmu_buf_will_dirty(ds_prev->ds_dbuf, tx);
|
|
|
|
if (after_branch_point &&
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds_prev)->ds_next_clones_obj != 0) {
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_dataset_remove_from_next_clones(ds_prev, obj, tx);
|
2015-04-01 15:14:34 +00:00
|
|
|
if (dsl_dataset_phys(ds)->ds_next_snap_obj != 0) {
|
2013-09-04 12:00:57 +00:00
|
|
|
VERIFY0(zap_add_int(mos,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds_prev)->
|
|
|
|
ds_next_clones_obj,
|
|
|
|
dsl_dataset_phys(ds)->ds_next_snap_obj,
|
|
|
|
tx));
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!after_branch_point) {
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds_prev)->ds_next_snap_obj =
|
|
|
|
dsl_dataset_phys(ds)->ds_next_snap_obj;
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-11-04 20:25:13 +00:00
|
|
|
dsl_dataset_t *ds_next;
|
|
|
|
uint64_t old_unique;
|
|
|
|
uint64_t used = 0, comp = 0, uncomp = 0;
|
|
|
|
|
2013-09-04 12:00:57 +00:00
|
|
|
VERIFY0(dsl_dataset_hold_obj(dp,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds)->ds_next_snap_obj, FTAG, &ds_next));
|
|
|
|
ASSERT3U(dsl_dataset_phys(ds_next)->ds_prev_snap_obj, ==, obj);
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2015-04-01 15:14:34 +00:00
|
|
|
old_unique = dsl_dataset_phys(ds_next)->ds_unique_bytes;
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
dmu_buf_will_dirty(ds_next->ds_dbuf, tx);
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds_next)->ds_prev_snap_obj =
|
|
|
|
dsl_dataset_phys(ds)->ds_prev_snap_obj;
|
|
|
|
dsl_dataset_phys(ds_next)->ds_prev_snap_txg =
|
|
|
|
dsl_dataset_phys(ds)->ds_prev_snap_txg;
|
|
|
|
ASSERT3U(dsl_dataset_phys(ds)->ds_prev_snap_txg, ==,
|
|
|
|
ds_prev ? dsl_dataset_phys(ds_prev)->ds_creation_txg : 0);
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
if (ds_next->ds_deadlist.dl_oldfmt) {
|
|
|
|
process_old_deadlist(ds, ds_prev, ds_next,
|
|
|
|
after_branch_point, tx);
|
|
|
|
} else {
|
|
|
|
/* Adjust prev's unique space. */
|
|
|
|
if (ds_prev && !after_branch_point) {
|
|
|
|
dsl_deadlist_space_range(&ds_next->ds_deadlist,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds_prev)->ds_prev_snap_txg,
|
|
|
|
dsl_dataset_phys(ds)->ds_prev_snap_txg,
|
2013-09-04 12:00:57 +00:00
|
|
|
&used, &comp, &uncomp);
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds_prev)->ds_unique_bytes += used;
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Adjust snapused. */
|
|
|
|
dsl_deadlist_space_range(&ds_next->ds_deadlist,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds)->ds_prev_snap_txg, UINT64_MAX,
|
2013-09-04 12:00:57 +00:00
|
|
|
&used, &comp, &uncomp);
|
|
|
|
dsl_dir_diduse_space(ds->ds_dir, DD_USED_SNAP,
|
|
|
|
-used, -comp, -uncomp, tx);
|
|
|
|
|
|
|
|
/* Move blocks to be freed to pool's free list. */
|
|
|
|
dsl_deadlist_move_bpobj(&ds_next->ds_deadlist,
|
2015-04-01 15:14:34 +00:00
|
|
|
&dp->dp_free_bpobj, dsl_dataset_phys(ds)->ds_prev_snap_txg,
|
2013-09-04 12:00:57 +00:00
|
|
|
tx);
|
|
|
|
dsl_dir_diduse_space(tx->tx_pool->dp_free_dir,
|
|
|
|
DD_USED_HEAD, used, comp, uncomp, tx);
|
|
|
|
|
|
|
|
/* Merge our deadlist into next's and free it. */
|
|
|
|
dsl_deadlist_merge(&ds_next->ds_deadlist,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds)->ds_deadlist_obj, tx);
|
Add fast path for zfs_ioc_space_snaps() handling of empty_bpobj
When there are many snapshots, calls to zfs_ioc_space_snaps() (e.g. from
`zfs destroy -nv pool/fs@snap1%snap10000`) can be very slow, resulting
in poor performance because we are holding the dp_config_rwlock the
entire time, blocking spa_sync() from continuing. With around ten
thousand snapshots, we've seen up to 500 seconds in this ioctl,
iterating over up to 50,000,000 bpobjs, ~99% of which are the empty
bpobj.
By creating a fast path for zfs_ioc_space_snaps() handling of the
empty_bpobj, we can achieve a ~5x performance improvement of this ioctl
(when there are many snapshots, and the deadlist is mostly
empty_bpobj's).
Reviewed-by: Pavel Zakharov <pavel.zakharov@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Paul Dagnelie <pcd@delphix.com>
Signed-off-by: Matthew Ahrens <mahrens@delphix.com>
External-issue: DLPX-58348
Closes #8744
2019-08-20 18:34:52 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* We are done with the deadlist tree (generated/used
|
|
|
|
* by dsl_deadlist_move_bpobj() and dsl_deadlist_merge()).
|
|
|
|
* Discard it to save memory.
|
|
|
|
*/
|
|
|
|
dsl_deadlist_discard_tree(&ds_next->ds_deadlist);
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
OpenZFS 7614, 9064 - zfs device evacuation/removal
OpenZFS 7614 - zfs device evacuation/removal
OpenZFS 9064 - remove_mirror should wait for device removal to complete
This project allows top-level vdevs to be removed from the storage pool
with "zpool remove", reducing the total amount of storage in the pool.
This operation copies all allocated regions of the device to be removed
onto other devices, recording the mapping from old to new location.
After the removal is complete, read and free operations to the removed
(now "indirect") vdev must be remapped and performed at the new location
on disk. The indirect mapping table is kept in memory whenever the pool
is loaded, so there is minimal performance overhead when doing operations
on the indirect vdev.
The size of the in-memory mapping table will be reduced when its entries
become "obsolete" because they are no longer used by any block pointers
in the pool. An entry becomes obsolete when all the blocks that use
it are freed. An entry can also become obsolete when all the snapshots
that reference it are deleted, and the block pointers that reference it
have been "remapped" in all filesystems/zvols (and clones). Whenever an
indirect block is written, all the block pointers in it will be "remapped"
to their new (concrete) locations if possible. This process can be
accelerated by using the "zfs remap" command to proactively rewrite all
indirect blocks that reference indirect (removed) vdevs.
Note that when a device is removed, we do not verify the checksum of
the data that is copied. This makes the process much faster, but if it
were used on redundant vdevs (i.e. mirror or raidz vdevs), it would be
possible to copy the wrong data, when we have the correct data on e.g.
the other side of the mirror.
At the moment, only mirrors and simple top-level vdevs can be removed
and no removal is allowed if any of the top-level vdevs are raidz.
Porting Notes:
* Avoid zero-sized kmem_alloc() in vdev_compact_children().
The device evacuation code adds a dependency that
vdev_compact_children() be able to properly empty the vdev_child
array by setting it to NULL and zeroing vdev_children. Under Linux,
kmem_alloc() and related functions return a sentinel pointer rather
than NULL for zero-sized allocations.
* Remove comment regarding "mpt" driver where zfs_remove_max_segment
is initialized to SPA_MAXBLOCKSIZE.
Change zfs_condense_indirect_commit_entry_delay_ticks to
zfs_condense_indirect_commit_entry_delay_ms for consistency with
most other tunables in which delays are specified in ms.
* ZTS changes:
Use set_tunable rather than mdb
Use zpool sync as appropriate
Use sync_pool instead of sync
Kill jobs during test_removal_with_operation to allow unmount/export
Don't add non-disk names such as "mirror" or "raidz" to $DISKS
Use $TEST_BASE_DIR instead of /tmp
Increase HZ from 100 to 1000 which is more common on Linux
removal_multiple_indirection.ksh
Reduce iterations in order to not time out on the code
coverage builders.
removal_resume_export:
Functionally, the test case is correct but there exists a race
where the kernel thread hasn't been fully started yet and is
not visible. Wait for up to 1 second for the removal thread
to be started before giving up on it. Also, increase the
amount of data copied in order that the removal not finish
before the export has a chance to fail.
* MMP compatibility, the concept of concrete versus non-concrete devices
has slightly changed the semantics of vdev_writeable(). Update
mmp_random_leaf_impl() accordingly.
* Updated dbuf_remap() to handle the org.zfsonlinux:large_dnode pool
feature which is not supported by OpenZFS.
* Added support for new vdev removal tracepoints.
* Test cases removal_with_zdb and removal_condense_export have been
intentionally disabled. When run manually they pass as intended,
but when running in the automated test environment they produce
unreliable results on the latest Fedora release.
They may work better once the upstream pool import refectoring is
merged into ZoL at which point they will be re-enabled.
Authored by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Alex Reece <alex@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Richard Laager <rlaager@wiktel.com>
Reviewed by: Tim Chase <tim@chase2k.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Approved by: Garrett D'Amore <garrett@damore.org>
Ported-by: Tim Chase <tim@chase2k.com>
Signed-off-by: Tim Chase <tim@chase2k.com>
OpenZFS-issue: https://www.illumos.org/issues/7614
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/f539f1eb
Closes #6900
2016-09-22 16:30:13 +00:00
|
|
|
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_deadlist_close(&ds->ds_deadlist);
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_deadlist_free(mos, dsl_dataset_phys(ds)->ds_deadlist_obj, tx);
|
2013-09-04 12:00:57 +00:00
|
|
|
dmu_buf_will_dirty(ds->ds_dbuf, tx);
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds)->ds_deadlist_obj = 0;
|
2013-09-04 12:00:57 +00:00
|
|
|
|
OpenZFS 7614, 9064 - zfs device evacuation/removal
OpenZFS 7614 - zfs device evacuation/removal
OpenZFS 9064 - remove_mirror should wait for device removal to complete
This project allows top-level vdevs to be removed from the storage pool
with "zpool remove", reducing the total amount of storage in the pool.
This operation copies all allocated regions of the device to be removed
onto other devices, recording the mapping from old to new location.
After the removal is complete, read and free operations to the removed
(now "indirect") vdev must be remapped and performed at the new location
on disk. The indirect mapping table is kept in memory whenever the pool
is loaded, so there is minimal performance overhead when doing operations
on the indirect vdev.
The size of the in-memory mapping table will be reduced when its entries
become "obsolete" because they are no longer used by any block pointers
in the pool. An entry becomes obsolete when all the blocks that use
it are freed. An entry can also become obsolete when all the snapshots
that reference it are deleted, and the block pointers that reference it
have been "remapped" in all filesystems/zvols (and clones). Whenever an
indirect block is written, all the block pointers in it will be "remapped"
to their new (concrete) locations if possible. This process can be
accelerated by using the "zfs remap" command to proactively rewrite all
indirect blocks that reference indirect (removed) vdevs.
Note that when a device is removed, we do not verify the checksum of
the data that is copied. This makes the process much faster, but if it
were used on redundant vdevs (i.e. mirror or raidz vdevs), it would be
possible to copy the wrong data, when we have the correct data on e.g.
the other side of the mirror.
At the moment, only mirrors and simple top-level vdevs can be removed
and no removal is allowed if any of the top-level vdevs are raidz.
Porting Notes:
* Avoid zero-sized kmem_alloc() in vdev_compact_children().
The device evacuation code adds a dependency that
vdev_compact_children() be able to properly empty the vdev_child
array by setting it to NULL and zeroing vdev_children. Under Linux,
kmem_alloc() and related functions return a sentinel pointer rather
than NULL for zero-sized allocations.
* Remove comment regarding "mpt" driver where zfs_remove_max_segment
is initialized to SPA_MAXBLOCKSIZE.
Change zfs_condense_indirect_commit_entry_delay_ticks to
zfs_condense_indirect_commit_entry_delay_ms for consistency with
most other tunables in which delays are specified in ms.
* ZTS changes:
Use set_tunable rather than mdb
Use zpool sync as appropriate
Use sync_pool instead of sync
Kill jobs during test_removal_with_operation to allow unmount/export
Don't add non-disk names such as "mirror" or "raidz" to $DISKS
Use $TEST_BASE_DIR instead of /tmp
Increase HZ from 100 to 1000 which is more common on Linux
removal_multiple_indirection.ksh
Reduce iterations in order to not time out on the code
coverage builders.
removal_resume_export:
Functionally, the test case is correct but there exists a race
where the kernel thread hasn't been fully started yet and is
not visible. Wait for up to 1 second for the removal thread
to be started before giving up on it. Also, increase the
amount of data copied in order that the removal not finish
before the export has a chance to fail.
* MMP compatibility, the concept of concrete versus non-concrete devices
has slightly changed the semantics of vdev_writeable(). Update
mmp_random_leaf_impl() accordingly.
* Updated dbuf_remap() to handle the org.zfsonlinux:large_dnode pool
feature which is not supported by OpenZFS.
* Added support for new vdev removal tracepoints.
* Test cases removal_with_zdb and removal_condense_export have been
intentionally disabled. When run manually they pass as intended,
but when running in the automated test environment they produce
unreliable results on the latest Fedora release.
They may work better once the upstream pool import refectoring is
merged into ZoL at which point they will be re-enabled.
Authored by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Alex Reece <alex@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Richard Laager <rlaager@wiktel.com>
Reviewed by: Tim Chase <tim@chase2k.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Approved by: Garrett D'Amore <garrett@damore.org>
Ported-by: Tim Chase <tim@chase2k.com>
Signed-off-by: Tim Chase <tim@chase2k.com>
OpenZFS-issue: https://www.illumos.org/issues/7614
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/f539f1eb
Closes #6900
2016-09-22 16:30:13 +00:00
|
|
|
dsl_destroy_snapshot_handle_remaps(ds, ds_next, tx);
|
|
|
|
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
if (!book_exists) {
|
|
|
|
/* Collapse range in clone heads */
|
|
|
|
dsl_dir_remove_clones_key(ds->ds_dir,
|
|
|
|
dsl_dataset_phys(ds)->ds_creation_txg, tx);
|
|
|
|
}
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2015-04-02 03:44:32 +00:00
|
|
|
if (ds_next->ds_is_snapshot) {
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_dataset_t *ds_nextnext;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Update next's unique to include blocks which
|
|
|
|
* were previously shared by only this snapshot
|
|
|
|
* and it. Those blocks will be born after the
|
|
|
|
* prev snap and before this snap, and will have
|
|
|
|
* died after the next snap and before the one
|
|
|
|
* after that (ie. be on the snap after next's
|
|
|
|
* deadlist).
|
|
|
|
*/
|
|
|
|
VERIFY0(dsl_dataset_hold_obj(dp,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds_next)->ds_next_snap_obj,
|
|
|
|
FTAG, &ds_nextnext));
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_deadlist_space_range(&ds_nextnext->ds_deadlist,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds)->ds_prev_snap_txg,
|
|
|
|
dsl_dataset_phys(ds)->ds_creation_txg,
|
2013-09-04 12:00:57 +00:00
|
|
|
&used, &comp, &uncomp);
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds_next)->ds_unique_bytes += used;
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_dataset_rele(ds_nextnext, FTAG);
|
|
|
|
ASSERT3P(ds_next->ds_prev, ==, NULL);
|
|
|
|
|
|
|
|
/* Collapse range in this head. */
|
2017-11-04 20:25:13 +00:00
|
|
|
dsl_dataset_t *hds;
|
2013-09-04 12:00:57 +00:00
|
|
|
VERIFY0(dsl_dataset_hold_obj(dp,
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
dsl_dir_phys(ds->ds_dir)->dd_head_dataset_obj,
|
|
|
|
FTAG, &hds));
|
|
|
|
if (!book_exists) {
|
|
|
|
/* Collapse range in this head. */
|
|
|
|
dsl_deadlist_remove_key(&hds->ds_deadlist,
|
|
|
|
dsl_dataset_phys(ds)->ds_creation_txg, tx);
|
|
|
|
}
|
OpenZFS 7614, 9064 - zfs device evacuation/removal
OpenZFS 7614 - zfs device evacuation/removal
OpenZFS 9064 - remove_mirror should wait for device removal to complete
This project allows top-level vdevs to be removed from the storage pool
with "zpool remove", reducing the total amount of storage in the pool.
This operation copies all allocated regions of the device to be removed
onto other devices, recording the mapping from old to new location.
After the removal is complete, read and free operations to the removed
(now "indirect") vdev must be remapped and performed at the new location
on disk. The indirect mapping table is kept in memory whenever the pool
is loaded, so there is minimal performance overhead when doing operations
on the indirect vdev.
The size of the in-memory mapping table will be reduced when its entries
become "obsolete" because they are no longer used by any block pointers
in the pool. An entry becomes obsolete when all the blocks that use
it are freed. An entry can also become obsolete when all the snapshots
that reference it are deleted, and the block pointers that reference it
have been "remapped" in all filesystems/zvols (and clones). Whenever an
indirect block is written, all the block pointers in it will be "remapped"
to their new (concrete) locations if possible. This process can be
accelerated by using the "zfs remap" command to proactively rewrite all
indirect blocks that reference indirect (removed) vdevs.
Note that when a device is removed, we do not verify the checksum of
the data that is copied. This makes the process much faster, but if it
were used on redundant vdevs (i.e. mirror or raidz vdevs), it would be
possible to copy the wrong data, when we have the correct data on e.g.
the other side of the mirror.
At the moment, only mirrors and simple top-level vdevs can be removed
and no removal is allowed if any of the top-level vdevs are raidz.
Porting Notes:
* Avoid zero-sized kmem_alloc() in vdev_compact_children().
The device evacuation code adds a dependency that
vdev_compact_children() be able to properly empty the vdev_child
array by setting it to NULL and zeroing vdev_children. Under Linux,
kmem_alloc() and related functions return a sentinel pointer rather
than NULL for zero-sized allocations.
* Remove comment regarding "mpt" driver where zfs_remove_max_segment
is initialized to SPA_MAXBLOCKSIZE.
Change zfs_condense_indirect_commit_entry_delay_ticks to
zfs_condense_indirect_commit_entry_delay_ms for consistency with
most other tunables in which delays are specified in ms.
* ZTS changes:
Use set_tunable rather than mdb
Use zpool sync as appropriate
Use sync_pool instead of sync
Kill jobs during test_removal_with_operation to allow unmount/export
Don't add non-disk names such as "mirror" or "raidz" to $DISKS
Use $TEST_BASE_DIR instead of /tmp
Increase HZ from 100 to 1000 which is more common on Linux
removal_multiple_indirection.ksh
Reduce iterations in order to not time out on the code
coverage builders.
removal_resume_export:
Functionally, the test case is correct but there exists a race
where the kernel thread hasn't been fully started yet and is
not visible. Wait for up to 1 second for the removal thread
to be started before giving up on it. Also, increase the
amount of data copied in order that the removal not finish
before the export has a chance to fail.
* MMP compatibility, the concept of concrete versus non-concrete devices
has slightly changed the semantics of vdev_writeable(). Update
mmp_random_leaf_impl() accordingly.
* Updated dbuf_remap() to handle the org.zfsonlinux:large_dnode pool
feature which is not supported by OpenZFS.
* Added support for new vdev removal tracepoints.
* Test cases removal_with_zdb and removal_condense_export have been
intentionally disabled. When run manually they pass as intended,
but when running in the automated test environment they produce
unreliable results on the latest Fedora release.
They may work better once the upstream pool import refectoring is
merged into ZoL at which point they will be re-enabled.
Authored by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Alex Reece <alex@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Richard Laager <rlaager@wiktel.com>
Reviewed by: Tim Chase <tim@chase2k.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Approved by: Garrett D'Amore <garrett@damore.org>
Ported-by: Tim Chase <tim@chase2k.com>
Signed-off-by: Tim Chase <tim@chase2k.com>
OpenZFS-issue: https://www.illumos.org/issues/7614
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/f539f1eb
Closes #6900
2016-09-22 16:30:13 +00:00
|
|
|
if (dsl_dataset_remap_deadlist_exists(hds)) {
|
|
|
|
dsl_deadlist_remove_key(&hds->ds_remap_deadlist,
|
|
|
|
dsl_dataset_phys(ds)->ds_creation_txg, tx);
|
|
|
|
}
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_dataset_rele(hds, FTAG);
|
|
|
|
|
|
|
|
} else {
|
|
|
|
ASSERT3P(ds_next->ds_prev, ==, ds);
|
|
|
|
dsl_dataset_rele(ds_next->ds_prev, ds_next);
|
|
|
|
ds_next->ds_prev = NULL;
|
|
|
|
if (ds_prev) {
|
|
|
|
VERIFY0(dsl_dataset_hold_obj(dp,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds)->ds_prev_snap_obj,
|
2013-09-04 12:00:57 +00:00
|
|
|
ds_next, &ds_next->ds_prev));
|
|
|
|
}
|
|
|
|
|
|
|
|
dsl_dataset_recalc_head_uniq(ds_next);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Reduce the amount of our unconsumed refreservation
|
|
|
|
* being charged to our parent by the amount of
|
|
|
|
* new unique data we have gained.
|
|
|
|
*/
|
|
|
|
if (old_unique < ds_next->ds_reserved) {
|
|
|
|
int64_t mrsdelta;
|
|
|
|
uint64_t new_unique =
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds_next)->ds_unique_bytes;
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
ASSERT(old_unique <= new_unique);
|
|
|
|
mrsdelta = MIN(new_unique - old_unique,
|
|
|
|
ds_next->ds_reserved - old_unique);
|
|
|
|
dsl_dir_diduse_space(ds->ds_dir,
|
|
|
|
DD_USED_REFRSRV, -mrsdelta, 0, 0, tx);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
dsl_dataset_rele(ds_next, FTAG);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This must be done after the dsl_traverse(), because it will
|
|
|
|
* re-open the objset.
|
|
|
|
*/
|
|
|
|
if (ds->ds_objset) {
|
|
|
|
dmu_objset_evict(ds->ds_objset);
|
|
|
|
ds->ds_objset = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* remove from snapshot namespace */
|
2017-11-04 20:25:13 +00:00
|
|
|
dsl_dataset_t *ds_head;
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT(dsl_dataset_phys(ds)->ds_snapnames_zapobj == 0);
|
2013-09-04 12:00:57 +00:00
|
|
|
VERIFY0(dsl_dataset_hold_obj(dp,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dir_phys(ds->ds_dir)->dd_head_dataset_obj, FTAG, &ds_head));
|
2013-09-04 12:00:57 +00:00
|
|
|
VERIFY0(dsl_dataset_get_snapname(ds));
|
|
|
|
#ifdef ZFS_DEBUG
|
|
|
|
{
|
|
|
|
uint64_t val;
|
2014-03-22 09:07:14 +00:00
|
|
|
int err;
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
err = dsl_dataset_snap_lookup(ds_head,
|
|
|
|
ds->ds_snapname, &val);
|
|
|
|
ASSERT0(err);
|
|
|
|
ASSERT3U(val, ==, obj);
|
|
|
|
}
|
|
|
|
#endif
|
2015-04-01 13:07:48 +00:00
|
|
|
VERIFY0(dsl_dataset_snap_remove(ds_head, ds->ds_snapname, tx, B_TRUE));
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_dataset_rele(ds_head, FTAG);
|
|
|
|
|
|
|
|
if (ds_prev != NULL)
|
|
|
|
dsl_dataset_rele(ds_prev, FTAG);
|
|
|
|
|
|
|
|
spa_prop_clear_bootfs(dp->dp_spa, ds->ds_object, tx);
|
|
|
|
|
2015-04-01 15:14:34 +00:00
|
|
|
if (dsl_dataset_phys(ds)->ds_next_clones_obj != 0) {
|
2019-12-05 20:37:00 +00:00
|
|
|
uint64_t count __maybe_unused;
|
2013-09-04 12:00:57 +00:00
|
|
|
ASSERT0(zap_count(mos,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds)->ds_next_clones_obj, &count) &&
|
|
|
|
count == 0);
|
2013-09-04 12:00:57 +00:00
|
|
|
VERIFY0(dmu_object_free(mos,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds)->ds_next_clones_obj, tx));
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
2015-04-01 15:14:34 +00:00
|
|
|
if (dsl_dataset_phys(ds)->ds_props_obj != 0)
|
|
|
|
VERIFY0(zap_destroy(mos, dsl_dataset_phys(ds)->ds_props_obj,
|
|
|
|
tx));
|
|
|
|
if (dsl_dataset_phys(ds)->ds_userrefs_obj != 0)
|
|
|
|
VERIFY0(zap_destroy(mos, dsl_dataset_phys(ds)->ds_userrefs_obj,
|
|
|
|
tx));
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_dir_rele(ds->ds_dir, ds);
|
|
|
|
ds->ds_dir = NULL;
|
2013-10-08 17:13:05 +00:00
|
|
|
dmu_object_free_zapified(mos, obj, tx);
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
|
2018-02-08 16:16:23 +00:00
|
|
|
void
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_destroy_snapshot_sync(void *arg, dmu_tx_t *tx)
|
|
|
|
{
|
2018-02-08 16:16:23 +00:00
|
|
|
dsl_destroy_snapshot_arg_t *ddsa = arg;
|
|
|
|
const char *dsname = ddsa->ddsa_name;
|
|
|
|
boolean_t defer = ddsa->ddsa_defer;
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2018-02-08 16:16:23 +00:00
|
|
|
dsl_pool_t *dp = dmu_tx_pool(tx);
|
|
|
|
dsl_dataset_t *ds;
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2018-02-08 16:16:23 +00:00
|
|
|
int error = dsl_dataset_hold(dp, dsname, FTAG, &ds);
|
|
|
|
if (error == ENOENT)
|
|
|
|
return;
|
|
|
|
ASSERT0(error);
|
|
|
|
dsl_destroy_snapshot_sync_impl(ds, defer, tx);
|
|
|
|
zvol_remove_minors(dp->dp_spa, dsname, B_TRUE);
|
|
|
|
dsl_dataset_rele(ds, FTAG);
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The semantics of this function are described in the comment above
|
|
|
|
* lzc_destroy_snaps(). To summarize:
|
|
|
|
*
|
|
|
|
* The snapshots must all be in the same pool.
|
|
|
|
*
|
|
|
|
* Snapshots that don't exist will be silently ignored (considered to be
|
|
|
|
* "already deleted").
|
|
|
|
*
|
|
|
|
* On success, all snaps will be destroyed and this will return 0.
|
|
|
|
* On failure, no snaps will be destroyed, the errlist will be filled in,
|
|
|
|
* and this will return an errno.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
dsl_destroy_snapshots_nvl(nvlist_t *snaps, boolean_t defer,
|
|
|
|
nvlist_t *errlist)
|
|
|
|
{
|
2018-02-08 16:16:23 +00:00
|
|
|
if (nvlist_next_nvpair(snaps, NULL) == NULL)
|
2013-09-04 12:00:57 +00:00
|
|
|
return (0);
|
|
|
|
|
2018-02-08 16:16:23 +00:00
|
|
|
/*
|
|
|
|
* lzc_destroy_snaps() is documented to take an nvlist whose
|
2018-02-08 16:32:45 +00:00
|
|
|
* values "don't matter". We need to convert that nvlist to
|
|
|
|
* one that we know can be converted to LUA. We also don't
|
|
|
|
* care about any duplicate entries because the nvlist will
|
|
|
|
* be converted to a LUA table which should take care of this.
|
2018-02-08 16:16:23 +00:00
|
|
|
*/
|
2018-02-08 16:32:45 +00:00
|
|
|
nvlist_t *snaps_normalized;
|
|
|
|
VERIFY0(nvlist_alloc(&snaps_normalized, 0, KM_SLEEP));
|
2018-02-08 16:16:23 +00:00
|
|
|
for (nvpair_t *pair = nvlist_next_nvpair(snaps, NULL);
|
|
|
|
pair != NULL; pair = nvlist_next_nvpair(snaps, pair)) {
|
|
|
|
fnvlist_add_boolean_value(snaps_normalized,
|
|
|
|
nvpair_name(pair), B_TRUE);
|
|
|
|
}
|
2018-02-08 16:32:45 +00:00
|
|
|
|
|
|
|
nvlist_t *arg;
|
|
|
|
VERIFY0(nvlist_alloc(&arg, 0, KM_SLEEP));
|
2018-02-08 16:16:23 +00:00
|
|
|
fnvlist_add_nvlist(arg, "snaps", snaps_normalized);
|
|
|
|
fnvlist_free(snaps_normalized);
|
|
|
|
fnvlist_add_boolean_value(arg, "defer", defer);
|
|
|
|
|
2018-02-08 16:32:45 +00:00
|
|
|
nvlist_t *wrapper;
|
|
|
|
VERIFY0(nvlist_alloc(&wrapper, 0, KM_SLEEP));
|
2018-02-08 16:16:23 +00:00
|
|
|
fnvlist_add_nvlist(wrapper, ZCP_ARG_ARGLIST, arg);
|
|
|
|
fnvlist_free(arg);
|
|
|
|
|
|
|
|
const char *program =
|
|
|
|
"arg = ...\n"
|
|
|
|
"snaps = arg['snaps']\n"
|
|
|
|
"defer = arg['defer']\n"
|
|
|
|
"errors = { }\n"
|
|
|
|
"has_errors = false\n"
|
|
|
|
"for snap, v in pairs(snaps) do\n"
|
|
|
|
" errno = zfs.check.destroy{snap, defer=defer}\n"
|
|
|
|
" zfs.debug('snap: ' .. snap .. ' errno: ' .. errno)\n"
|
|
|
|
" if errno == ENOENT then\n"
|
|
|
|
" snaps[snap] = nil\n"
|
|
|
|
" elseif errno ~= 0 then\n"
|
|
|
|
" errors[snap] = errno\n"
|
|
|
|
" has_errors = true\n"
|
|
|
|
" end\n"
|
|
|
|
"end\n"
|
|
|
|
"if has_errors then\n"
|
|
|
|
" return errors\n"
|
|
|
|
"end\n"
|
|
|
|
"for snap, v in pairs(snaps) do\n"
|
|
|
|
" errno = zfs.sync.destroy{snap, defer=defer}\n"
|
|
|
|
" assert(errno == 0)\n"
|
|
|
|
"end\n"
|
|
|
|
"return { }\n";
|
|
|
|
|
|
|
|
nvlist_t *result = fnvlist_alloc();
|
|
|
|
int error = zcp_eval(nvpair_name(nvlist_next_nvpair(snaps, NULL)),
|
|
|
|
program,
|
2018-02-08 16:35:09 +00:00
|
|
|
B_TRUE,
|
2018-02-08 16:16:23 +00:00
|
|
|
0,
|
|
|
|
zfs_lua_max_memlimit,
|
2018-02-08 16:32:45 +00:00
|
|
|
nvlist_next_nvpair(wrapper, NULL), result);
|
2018-02-08 16:16:23 +00:00
|
|
|
if (error != 0) {
|
|
|
|
char *errorstr = NULL;
|
|
|
|
(void) nvlist_lookup_string(result, ZCP_RET_ERROR, &errorstr);
|
|
|
|
if (errorstr != NULL) {
|
|
|
|
zfs_dbgmsg(errorstr);
|
|
|
|
}
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
fnvlist_free(wrapper);
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2018-02-08 16:16:23 +00:00
|
|
|
/*
|
|
|
|
* lzc_destroy_snaps() is documented to fill the errlist with
|
2019-09-03 00:56:41 +00:00
|
|
|
* int32 values, so we need to convert the int64 values that are
|
2018-02-08 16:16:23 +00:00
|
|
|
* returned from LUA.
|
|
|
|
*/
|
|
|
|
int rv = 0;
|
|
|
|
nvlist_t *errlist_raw = fnvlist_lookup_nvlist(result, ZCP_RET_RETURN);
|
|
|
|
for (nvpair_t *pair = nvlist_next_nvpair(errlist_raw, NULL);
|
|
|
|
pair != NULL; pair = nvlist_next_nvpair(errlist_raw, pair)) {
|
|
|
|
int32_t val = (int32_t)fnvpair_value_int64(pair);
|
|
|
|
if (rv == 0)
|
|
|
|
rv = val;
|
|
|
|
fnvlist_add_int32(errlist, nvpair_name(pair), val);
|
|
|
|
}
|
|
|
|
fnvlist_free(result);
|
|
|
|
return (rv);
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
dsl_destroy_snapshot(const char *name, boolean_t defer)
|
|
|
|
{
|
|
|
|
int error;
|
2014-11-21 00:09:39 +00:00
|
|
|
nvlist_t *nvl = fnvlist_alloc();
|
|
|
|
nvlist_t *errlist = fnvlist_alloc();
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
fnvlist_add_boolean(nvl, name);
|
|
|
|
error = dsl_destroy_snapshots_nvl(nvl, defer, errlist);
|
|
|
|
fnvlist_free(errlist);
|
|
|
|
fnvlist_free(nvl);
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
|
|
|
struct killarg {
|
|
|
|
dsl_dataset_t *ds;
|
|
|
|
dmu_tx_t *tx;
|
|
|
|
};
|
|
|
|
|
|
|
|
/* ARGSUSED */
|
|
|
|
static int
|
|
|
|
kill_blkptr(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
|
2014-06-25 18:37:59 +00:00
|
|
|
const zbookmark_phys_t *zb, const dnode_phys_t *dnp, void *arg)
|
2013-09-04 12:00:57 +00:00
|
|
|
{
|
|
|
|
struct killarg *ka = arg;
|
|
|
|
dmu_tx_t *tx = ka->tx;
|
|
|
|
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
if (zb->zb_level == ZB_DNODE_LEVEL || BP_IS_HOLE(bp) ||
|
|
|
|
BP_IS_EMBEDDED(bp))
|
2013-09-04 12:00:57 +00:00
|
|
|
return (0);
|
|
|
|
|
|
|
|
if (zb->zb_level == ZB_ZIL_LEVEL) {
|
|
|
|
ASSERT(zilog != NULL);
|
|
|
|
/*
|
|
|
|
* It's a block in the intent log. It has no
|
|
|
|
* accounting, so just free it.
|
|
|
|
*/
|
|
|
|
dsl_free(ka->tx->tx_pool, ka->tx->tx_txg, bp);
|
|
|
|
} else {
|
|
|
|
ASSERT(zilog == NULL);
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT3U(bp->blk_birth, >,
|
|
|
|
dsl_dataset_phys(ka->ds)->ds_prev_snap_txg);
|
2013-09-04 12:00:57 +00:00
|
|
|
(void) dsl_dataset_block_kill(ka->ds, bp, tx, B_FALSE);
|
|
|
|
}
|
|
|
|
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
old_synchronous_dataset_destroy(dsl_dataset_t *ds, dmu_tx_t *tx)
|
|
|
|
{
|
|
|
|
struct killarg ka;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Free everything that we point to (that's born after
|
|
|
|
* the previous snapshot, if we are a clone)
|
|
|
|
*
|
|
|
|
* NB: this should be very quick, because we already
|
|
|
|
* freed all the objects in open context.
|
|
|
|
*/
|
|
|
|
ka.ds = ds;
|
|
|
|
ka.tx = tx;
|
|
|
|
VERIFY0(traverse_dataset(ds,
|
Native Encryption for ZFS on Linux
This change incorporates three major pieces:
The first change is a keystore that manages wrapping
and encryption keys for encrypted datasets. These
commands mostly involve manipulating the new
DSL Crypto Key ZAP Objects that live in the MOS. Each
encrypted dataset has its own DSL Crypto Key that is
protected with a user's key. This level of indirection
allows users to change their keys without re-encrypting
their entire datasets. The change implements the new
subcommands "zfs load-key", "zfs unload-key" and
"zfs change-key" which allow the user to manage their
encryption keys and settings. In addition, several new
flags and properties have been added to allow dataset
creation and to make mounting and unmounting more
convenient.
The second piece of this patch provides the ability to
encrypt, decyrpt, and authenticate protected datasets.
Each object set maintains a Merkel tree of Message
Authentication Codes that protect the lower layers,
similarly to how checksums are maintained. This part
impacts the zio layer, which handles the actual
encryption and generation of MACs, as well as the ARC
and DMU, which need to be able to handle encrypted
buffers and protected data.
The last addition is the ability to do raw, encrypted
sends and receives. The idea here is to send raw
encrypted and compressed data and receive it exactly
as is on a backup system. This means that the dataset
on the receiving system is protected using the same
user key that is in use on the sending side. By doing
so, datasets can be efficiently backed up to an
untrusted system without fear of data being
compromised.
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Jorgen Lundman <lundman@lundman.net>
Signed-off-by: Tom Caputi <tcaputi@datto.com>
Closes #494
Closes #5769
2017-08-14 17:36:48 +00:00
|
|
|
dsl_dataset_phys(ds)->ds_prev_snap_txg, TRAVERSE_POST |
|
|
|
|
TRAVERSE_NO_DECRYPT, kill_blkptr, &ka));
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT(!DS_UNIQUE_IS_ACCURATE(ds) ||
|
|
|
|
dsl_dataset_phys(ds)->ds_unique_bytes == 0);
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
dsl_destroy_head_check_impl(dsl_dataset_t *ds, int expected_holds)
|
|
|
|
{
|
|
|
|
int error;
|
|
|
|
uint64_t count;
|
|
|
|
objset_t *mos;
|
|
|
|
|
2015-04-02 03:44:32 +00:00
|
|
|
ASSERT(!ds->ds_is_snapshot);
|
|
|
|
if (ds->ds_is_snapshot)
|
2013-03-08 18:41:28 +00:00
|
|
|
return (SET_ERROR(EINVAL));
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2018-10-01 17:42:05 +00:00
|
|
|
if (zfs_refcount_count(&ds->ds_longholds) != expected_holds)
|
2013-03-08 18:41:28 +00:00
|
|
|
return (SET_ERROR(EBUSY));
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
mos = ds->ds_dir->dd_pool->dp_meta_objset;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Can't delete a head dataset if there are snapshots of it.
|
|
|
|
* (Except if the only snapshots are from the branch we cloned
|
|
|
|
* from.)
|
|
|
|
*/
|
|
|
|
if (ds->ds_prev != NULL &&
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj == ds->ds_object)
|
2013-03-08 18:41:28 +00:00
|
|
|
return (SET_ERROR(EBUSY));
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Can't delete if there are children of this fs.
|
|
|
|
*/
|
|
|
|
error = zap_count(mos,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dir_phys(ds->ds_dir)->dd_child_dir_zapobj, &count);
|
2013-09-04 12:00:57 +00:00
|
|
|
if (error != 0)
|
|
|
|
return (error);
|
|
|
|
if (count != 0)
|
2013-03-08 18:41:28 +00:00
|
|
|
return (SET_ERROR(EEXIST));
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
if (dsl_dir_is_clone(ds->ds_dir) && DS_IS_DEFER_DESTROY(ds->ds_prev) &&
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds->ds_prev)->ds_num_children == 2 &&
|
2013-09-04 12:00:57 +00:00
|
|
|
ds->ds_prev->ds_userrefs == 0) {
|
|
|
|
/* We need to remove the origin snapshot as well. */
|
2018-10-01 17:42:05 +00:00
|
|
|
if (!zfs_refcount_is_zero(&ds->ds_prev->ds_longholds))
|
2013-03-08 18:41:28 +00:00
|
|
|
return (SET_ERROR(EBUSY));
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
2018-02-08 16:16:23 +00:00
|
|
|
int
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_destroy_head_check(void *arg, dmu_tx_t *tx)
|
|
|
|
{
|
|
|
|
dsl_destroy_head_arg_t *ddha = arg;
|
|
|
|
dsl_pool_t *dp = dmu_tx_pool(tx);
|
|
|
|
dsl_dataset_t *ds;
|
|
|
|
int error;
|
|
|
|
|
|
|
|
error = dsl_dataset_hold(dp, ddha->ddha_name, FTAG, &ds);
|
|
|
|
if (error != 0)
|
|
|
|
return (error);
|
|
|
|
|
|
|
|
error = dsl_destroy_head_check_impl(ds, 0);
|
|
|
|
dsl_dataset_rele(ds, FTAG);
|
|
|
|
return (error);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
dsl_dir_destroy_sync(uint64_t ddobj, dmu_tx_t *tx)
|
|
|
|
{
|
|
|
|
dsl_dir_t *dd;
|
|
|
|
dsl_pool_t *dp = dmu_tx_pool(tx);
|
|
|
|
objset_t *mos = dp->dp_meta_objset;
|
|
|
|
dd_used_t t;
|
|
|
|
|
|
|
|
ASSERT(RRW_WRITE_HELD(&dmu_tx_pool(tx)->dp_config_rwlock));
|
|
|
|
|
|
|
|
VERIFY0(dsl_dir_hold_obj(dp, ddobj, NULL, FTAG, &dd));
|
|
|
|
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT0(dsl_dir_phys(dd)->dd_head_dataset_obj);
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2019-01-08 18:17:46 +00:00
|
|
|
/* Decrement the filesystem count for all parent filesystems. */
|
|
|
|
if (dd->dd_parent != NULL)
|
2015-04-01 13:07:48 +00:00
|
|
|
dsl_fs_ss_count_adjust(dd->dd_parent, -1,
|
|
|
|
DD_FIELD_FILESYSTEM_COUNT, tx);
|
|
|
|
|
2013-09-04 12:00:57 +00:00
|
|
|
/*
|
|
|
|
* Remove our reservation. The impl() routine avoids setting the
|
|
|
|
* actual property, which would require the (already destroyed) ds.
|
|
|
|
*/
|
|
|
|
dsl_dir_set_reservation_sync_impl(dd, 0, tx);
|
|
|
|
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT0(dsl_dir_phys(dd)->dd_used_bytes);
|
|
|
|
ASSERT0(dsl_dir_phys(dd)->dd_reserved);
|
2013-09-04 12:00:57 +00:00
|
|
|
for (t = 0; t < DD_USED_NUM; t++)
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT0(dsl_dir_phys(dd)->dd_used_breakdown[t]);
|
2013-09-04 12:00:57 +00:00
|
|
|
|
Native Encryption for ZFS on Linux
This change incorporates three major pieces:
The first change is a keystore that manages wrapping
and encryption keys for encrypted datasets. These
commands mostly involve manipulating the new
DSL Crypto Key ZAP Objects that live in the MOS. Each
encrypted dataset has its own DSL Crypto Key that is
protected with a user's key. This level of indirection
allows users to change their keys without re-encrypting
their entire datasets. The change implements the new
subcommands "zfs load-key", "zfs unload-key" and
"zfs change-key" which allow the user to manage their
encryption keys and settings. In addition, several new
flags and properties have been added to allow dataset
creation and to make mounting and unmounting more
convenient.
The second piece of this patch provides the ability to
encrypt, decyrpt, and authenticate protected datasets.
Each object set maintains a Merkel tree of Message
Authentication Codes that protect the lower layers,
similarly to how checksums are maintained. This part
impacts the zio layer, which handles the actual
encryption and generation of MACs, as well as the ARC
and DMU, which need to be able to handle encrypted
buffers and protected data.
The last addition is the ability to do raw, encrypted
sends and receives. The idea here is to send raw
encrypted and compressed data and receive it exactly
as is on a backup system. This means that the dataset
on the receiving system is protected using the same
user key that is in use on the sending side. By doing
so, datasets can be efficiently backed up to an
untrusted system without fear of data being
compromised.
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Jorgen Lundman <lundman@lundman.net>
Signed-off-by: Tom Caputi <tcaputi@datto.com>
Closes #494
Closes #5769
2017-08-14 17:36:48 +00:00
|
|
|
if (dd->dd_crypto_obj != 0) {
|
|
|
|
dsl_crypto_key_destroy_sync(dd->dd_crypto_obj, tx);
|
|
|
|
(void) spa_keystore_unload_wkey_impl(dp->dp_spa, dd->dd_object);
|
|
|
|
}
|
|
|
|
|
2015-04-01 15:14:34 +00:00
|
|
|
VERIFY0(zap_destroy(mos, dsl_dir_phys(dd)->dd_child_dir_zapobj, tx));
|
|
|
|
VERIFY0(zap_destroy(mos, dsl_dir_phys(dd)->dd_props_zapobj, tx));
|
2018-10-12 18:28:26 +00:00
|
|
|
if (dsl_dir_phys(dd)->dd_clones != 0)
|
|
|
|
VERIFY0(zap_destroy(mos, dsl_dir_phys(dd)->dd_clones, tx));
|
2015-04-01 15:14:34 +00:00
|
|
|
VERIFY0(dsl_deleg_destroy(mos, dsl_dir_phys(dd)->dd_deleg_zapobj, tx));
|
2013-09-04 12:00:57 +00:00
|
|
|
VERIFY0(zap_remove(mos,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dir_phys(dd->dd_parent)->dd_child_dir_zapobj,
|
|
|
|
dd->dd_myname, tx));
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
dsl_dir_rele(dd, FTAG);
|
2013-10-08 17:13:05 +00:00
|
|
|
dmu_object_free_zapified(mos, ddobj, tx);
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
|
2019-07-26 17:54:14 +00:00
|
|
|
static void
|
|
|
|
dsl_clone_destroy_assert(dsl_dir_t *dd)
|
|
|
|
{
|
|
|
|
uint64_t used, comp, uncomp;
|
|
|
|
|
|
|
|
ASSERT(dsl_dir_is_clone(dd));
|
|
|
|
dsl_deadlist_space(&dd->dd_livelist, &used, &comp, &uncomp);
|
|
|
|
|
|
|
|
ASSERT3U(dsl_dir_phys(dd)->dd_used_bytes, ==, used);
|
|
|
|
ASSERT3U(dsl_dir_phys(dd)->dd_compressed_bytes, ==, comp);
|
|
|
|
/*
|
|
|
|
* Greater than because we do not track embedded block pointers in
|
|
|
|
* the livelist
|
|
|
|
*/
|
|
|
|
ASSERT3U(dsl_dir_phys(dd)->dd_uncompressed_bytes, >=, uncomp);
|
|
|
|
|
|
|
|
ASSERT(list_is_empty(&dd->dd_pending_allocs.bpl_list));
|
|
|
|
ASSERT(list_is_empty(&dd->dd_pending_frees.bpl_list));
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Start the delete process for a clone. Free its zil, verify the space usage
|
|
|
|
* and queue the blkptrs for deletion by adding the livelist to the pool-wide
|
|
|
|
* delete queue.
|
|
|
|
*/
|
|
|
|
static void
|
|
|
|
dsl_async_clone_destroy(dsl_dataset_t *ds, dmu_tx_t *tx)
|
|
|
|
{
|
|
|
|
uint64_t zap_obj, to_delete, used, comp, uncomp;
|
|
|
|
objset_t *os;
|
|
|
|
dsl_dir_t *dd = ds->ds_dir;
|
|
|
|
dsl_pool_t *dp = dmu_tx_pool(tx);
|
|
|
|
objset_t *mos = dp->dp_meta_objset;
|
|
|
|
spa_t *spa = dmu_tx_pool(tx)->dp_spa;
|
|
|
|
VERIFY0(dmu_objset_from_ds(ds, &os));
|
|
|
|
|
|
|
|
/* Check that the clone is in a correct state to be deleted */
|
|
|
|
dsl_clone_destroy_assert(dd);
|
|
|
|
|
|
|
|
/* Destroy the zil */
|
|
|
|
zil_destroy_sync(dmu_objset_zil(os), tx);
|
|
|
|
|
|
|
|
VERIFY0(zap_lookup(mos, dd->dd_object,
|
|
|
|
DD_FIELD_LIVELIST, sizeof (uint64_t), 1, &to_delete));
|
|
|
|
/* Initialize deleted_clones entry to track livelists to cleanup */
|
|
|
|
int error = zap_lookup(mos, DMU_POOL_DIRECTORY_OBJECT,
|
|
|
|
DMU_POOL_DELETED_CLONES, sizeof (uint64_t), 1, &zap_obj);
|
|
|
|
if (error == ENOENT) {
|
|
|
|
zap_obj = zap_create(mos, DMU_OTN_ZAP_METADATA,
|
|
|
|
DMU_OT_NONE, 0, tx);
|
|
|
|
VERIFY0(zap_add(mos, DMU_POOL_DIRECTORY_OBJECT,
|
|
|
|
DMU_POOL_DELETED_CLONES, sizeof (uint64_t), 1,
|
|
|
|
&(zap_obj), tx));
|
|
|
|
spa->spa_livelists_to_delete = zap_obj;
|
|
|
|
} else if (error != 0) {
|
|
|
|
zfs_panic_recover("zfs: error %d was returned while looking "
|
|
|
|
"up DMU_POOL_DELETED_CLONES in the zap");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
VERIFY0(zap_add_int(mos, zap_obj, to_delete, tx));
|
|
|
|
|
|
|
|
/* Clone is no longer using space, now tracked by dp_free_dir */
|
|
|
|
dsl_deadlist_space(&dd->dd_livelist, &used, &comp, &uncomp);
|
|
|
|
dsl_dir_diduse_space(dd, DD_USED_HEAD,
|
|
|
|
-used, -comp, -dsl_dir_phys(dd)->dd_uncompressed_bytes,
|
|
|
|
tx);
|
|
|
|
dsl_dir_diduse_space(dp->dp_free_dir, DD_USED_HEAD,
|
|
|
|
used, comp, uncomp, tx);
|
|
|
|
dsl_dir_remove_livelist(dd, tx, B_FALSE);
|
|
|
|
zthr_wakeup(spa->spa_livelist_delete_zthr);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Move the bptree into the pool's list of trees to clean up, update space
|
|
|
|
* accounting information and destroy the zil.
|
|
|
|
*/
|
2019-10-17 01:34:19 +00:00
|
|
|
static void
|
2019-07-26 17:54:14 +00:00
|
|
|
dsl_async_dataset_destroy(dsl_dataset_t *ds, dmu_tx_t *tx)
|
|
|
|
{
|
|
|
|
uint64_t used, comp, uncomp;
|
|
|
|
objset_t *os;
|
|
|
|
|
|
|
|
VERIFY0(dmu_objset_from_ds(ds, &os));
|
|
|
|
dsl_pool_t *dp = dmu_tx_pool(tx);
|
|
|
|
objset_t *mos = dp->dp_meta_objset;
|
|
|
|
|
|
|
|
zil_destroy_sync(dmu_objset_zil(os), tx);
|
|
|
|
|
|
|
|
if (!spa_feature_is_active(dp->dp_spa,
|
|
|
|
SPA_FEATURE_ASYNC_DESTROY)) {
|
|
|
|
dsl_scan_t *scn = dp->dp_scan;
|
|
|
|
spa_feature_incr(dp->dp_spa, SPA_FEATURE_ASYNC_DESTROY,
|
|
|
|
tx);
|
|
|
|
dp->dp_bptree_obj = bptree_alloc(mos, tx);
|
|
|
|
VERIFY0(zap_add(mos,
|
|
|
|
DMU_POOL_DIRECTORY_OBJECT,
|
|
|
|
DMU_POOL_BPTREE_OBJ, sizeof (uint64_t), 1,
|
|
|
|
&dp->dp_bptree_obj, tx));
|
|
|
|
ASSERT(!scn->scn_async_destroying);
|
|
|
|
scn->scn_async_destroying = B_TRUE;
|
|
|
|
}
|
|
|
|
|
|
|
|
used = dsl_dir_phys(ds->ds_dir)->dd_used_bytes;
|
|
|
|
comp = dsl_dir_phys(ds->ds_dir)->dd_compressed_bytes;
|
|
|
|
uncomp = dsl_dir_phys(ds->ds_dir)->dd_uncompressed_bytes;
|
|
|
|
|
|
|
|
ASSERT(!DS_UNIQUE_IS_ACCURATE(ds) ||
|
|
|
|
dsl_dataset_phys(ds)->ds_unique_bytes == used);
|
|
|
|
|
|
|
|
rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
|
|
|
|
bptree_add(mos, dp->dp_bptree_obj,
|
|
|
|
&dsl_dataset_phys(ds)->ds_bp,
|
|
|
|
dsl_dataset_phys(ds)->ds_prev_snap_txg,
|
|
|
|
used, comp, uncomp, tx);
|
|
|
|
rrw_exit(&ds->ds_bp_rwlock, FTAG);
|
|
|
|
dsl_dir_diduse_space(ds->ds_dir, DD_USED_HEAD,
|
|
|
|
-used, -comp, -uncomp, tx);
|
|
|
|
dsl_dir_diduse_space(dp->dp_free_dir, DD_USED_HEAD,
|
|
|
|
used, comp, uncomp, tx);
|
|
|
|
}
|
|
|
|
|
2013-09-04 12:00:57 +00:00
|
|
|
void
|
|
|
|
dsl_destroy_head_sync_impl(dsl_dataset_t *ds, dmu_tx_t *tx)
|
|
|
|
{
|
|
|
|
dsl_pool_t *dp = dmu_tx_pool(tx);
|
|
|
|
objset_t *mos = dp->dp_meta_objset;
|
|
|
|
uint64_t obj, ddobj, prevobj = 0;
|
|
|
|
boolean_t rmorigin;
|
|
|
|
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT3U(dsl_dataset_phys(ds)->ds_num_children, <=, 1);
|
2013-09-04 12:00:57 +00:00
|
|
|
ASSERT(ds->ds_prev == NULL ||
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj != ds->ds_object);
|
2017-01-27 19:43:42 +00:00
|
|
|
rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT3U(dsl_dataset_phys(ds)->ds_bp.blk_birth, <=, tx->tx_txg);
|
2017-01-27 19:43:42 +00:00
|
|
|
rrw_exit(&ds->ds_bp_rwlock, FTAG);
|
2013-09-04 12:00:57 +00:00
|
|
|
ASSERT(RRW_WRITE_HELD(&dp->dp_config_rwlock));
|
|
|
|
|
|
|
|
/* We need to log before removing it from the namespace. */
|
2019-09-12 20:28:26 +00:00
|
|
|
spa_history_log_internal_ds(ds, "destroy", tx, " ");
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
rmorigin = (dsl_dir_is_clone(ds->ds_dir) &&
|
|
|
|
DS_IS_DEFER_DESTROY(ds->ds_prev) &&
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds->ds_prev)->ds_num_children == 2 &&
|
2013-09-04 12:00:57 +00:00
|
|
|
ds->ds_prev->ds_userrefs == 0);
|
|
|
|
|
2014-06-05 21:19:08 +00:00
|
|
|
/* Remove our reservation. */
|
2013-09-04 12:00:57 +00:00
|
|
|
if (ds->ds_reserved != 0) {
|
|
|
|
dsl_dataset_set_refreservation_sync_impl(ds,
|
|
|
|
(ZPROP_SRC_NONE | ZPROP_SRC_LOCAL | ZPROP_SRC_RECEIVED),
|
|
|
|
0, tx);
|
|
|
|
ASSERT0(ds->ds_reserved);
|
|
|
|
}
|
|
|
|
|
2015-07-24 16:53:55 +00:00
|
|
|
obj = ds->ds_object;
|
2014-11-03 20:15:08 +00:00
|
|
|
|
2017-11-04 20:25:13 +00:00
|
|
|
for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
|
2018-10-16 18:15:04 +00:00
|
|
|
if (dsl_dataset_feature_is_active(ds, f))
|
|
|
|
dsl_dataset_deactivate_feature(ds, f, tx);
|
2015-07-24 16:53:55 +00:00
|
|
|
}
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2015-07-24 16:53:55 +00:00
|
|
|
dsl_scan_ds_destroyed(ds, tx);
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2015-04-01 15:14:34 +00:00
|
|
|
if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
|
2013-09-04 12:00:57 +00:00
|
|
|
/* This is a clone */
|
|
|
|
ASSERT(ds->ds_prev != NULL);
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT3U(dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj, !=,
|
|
|
|
obj);
|
|
|
|
ASSERT0(dsl_dataset_phys(ds)->ds_next_snap_obj);
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
|
2015-04-01 15:14:34 +00:00
|
|
|
if (dsl_dataset_phys(ds->ds_prev)->ds_next_clones_obj != 0) {
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_dataset_remove_from_next_clones(ds->ds_prev,
|
|
|
|
obj, tx);
|
|
|
|
}
|
|
|
|
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT3U(dsl_dataset_phys(ds->ds_prev)->ds_num_children, >, 1);
|
|
|
|
dsl_dataset_phys(ds->ds_prev)->ds_num_children--;
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2019-07-26 17:54:14 +00:00
|
|
|
* Destroy the deadlist. Unless it's a clone, the
|
OpenZFS 7614, 9064 - zfs device evacuation/removal
OpenZFS 7614 - zfs device evacuation/removal
OpenZFS 9064 - remove_mirror should wait for device removal to complete
This project allows top-level vdevs to be removed from the storage pool
with "zpool remove", reducing the total amount of storage in the pool.
This operation copies all allocated regions of the device to be removed
onto other devices, recording the mapping from old to new location.
After the removal is complete, read and free operations to the removed
(now "indirect") vdev must be remapped and performed at the new location
on disk. The indirect mapping table is kept in memory whenever the pool
is loaded, so there is minimal performance overhead when doing operations
on the indirect vdev.
The size of the in-memory mapping table will be reduced when its entries
become "obsolete" because they are no longer used by any block pointers
in the pool. An entry becomes obsolete when all the blocks that use
it are freed. An entry can also become obsolete when all the snapshots
that reference it are deleted, and the block pointers that reference it
have been "remapped" in all filesystems/zvols (and clones). Whenever an
indirect block is written, all the block pointers in it will be "remapped"
to their new (concrete) locations if possible. This process can be
accelerated by using the "zfs remap" command to proactively rewrite all
indirect blocks that reference indirect (removed) vdevs.
Note that when a device is removed, we do not verify the checksum of
the data that is copied. This makes the process much faster, but if it
were used on redundant vdevs (i.e. mirror or raidz vdevs), it would be
possible to copy the wrong data, when we have the correct data on e.g.
the other side of the mirror.
At the moment, only mirrors and simple top-level vdevs can be removed
and no removal is allowed if any of the top-level vdevs are raidz.
Porting Notes:
* Avoid zero-sized kmem_alloc() in vdev_compact_children().
The device evacuation code adds a dependency that
vdev_compact_children() be able to properly empty the vdev_child
array by setting it to NULL and zeroing vdev_children. Under Linux,
kmem_alloc() and related functions return a sentinel pointer rather
than NULL for zero-sized allocations.
* Remove comment regarding "mpt" driver where zfs_remove_max_segment
is initialized to SPA_MAXBLOCKSIZE.
Change zfs_condense_indirect_commit_entry_delay_ticks to
zfs_condense_indirect_commit_entry_delay_ms for consistency with
most other tunables in which delays are specified in ms.
* ZTS changes:
Use set_tunable rather than mdb
Use zpool sync as appropriate
Use sync_pool instead of sync
Kill jobs during test_removal_with_operation to allow unmount/export
Don't add non-disk names such as "mirror" or "raidz" to $DISKS
Use $TEST_BASE_DIR instead of /tmp
Increase HZ from 100 to 1000 which is more common on Linux
removal_multiple_indirection.ksh
Reduce iterations in order to not time out on the code
coverage builders.
removal_resume_export:
Functionally, the test case is correct but there exists a race
where the kernel thread hasn't been fully started yet and is
not visible. Wait for up to 1 second for the removal thread
to be started before giving up on it. Also, increase the
amount of data copied in order that the removal not finish
before the export has a chance to fail.
* MMP compatibility, the concept of concrete versus non-concrete devices
has slightly changed the semantics of vdev_writeable(). Update
mmp_random_leaf_impl() accordingly.
* Updated dbuf_remap() to handle the org.zfsonlinux:large_dnode pool
feature which is not supported by OpenZFS.
* Added support for new vdev removal tracepoints.
* Test cases removal_with_zdb and removal_condense_export have been
intentionally disabled. When run manually they pass as intended,
but when running in the automated test environment they produce
unreliable results on the latest Fedora release.
They may work better once the upstream pool import refectoring is
merged into ZoL at which point they will be re-enabled.
Authored by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Alex Reece <alex@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Richard Laager <rlaager@wiktel.com>
Reviewed by: Tim Chase <tim@chase2k.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Approved by: Garrett D'Amore <garrett@damore.org>
Ported-by: Tim Chase <tim@chase2k.com>
Signed-off-by: Tim Chase <tim@chase2k.com>
OpenZFS-issue: https://www.illumos.org/issues/7614
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/f539f1eb
Closes #6900
2016-09-22 16:30:13 +00:00
|
|
|
* deadlist should be empty since the dataset has no snapshots.
|
|
|
|
* (If it's a clone, it's safe to ignore the deadlist contents
|
|
|
|
* since they are still referenced by the origin snapshot.)
|
2013-09-04 12:00:57 +00:00
|
|
|
*/
|
|
|
|
dsl_deadlist_close(&ds->ds_deadlist);
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_deadlist_free(mos, dsl_dataset_phys(ds)->ds_deadlist_obj, tx);
|
2013-09-04 12:00:57 +00:00
|
|
|
dmu_buf_will_dirty(ds->ds_dbuf, tx);
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds)->ds_deadlist_obj = 0;
|
2013-09-04 12:00:57 +00:00
|
|
|
|
OpenZFS 7614, 9064 - zfs device evacuation/removal
OpenZFS 7614 - zfs device evacuation/removal
OpenZFS 9064 - remove_mirror should wait for device removal to complete
This project allows top-level vdevs to be removed from the storage pool
with "zpool remove", reducing the total amount of storage in the pool.
This operation copies all allocated regions of the device to be removed
onto other devices, recording the mapping from old to new location.
After the removal is complete, read and free operations to the removed
(now "indirect") vdev must be remapped and performed at the new location
on disk. The indirect mapping table is kept in memory whenever the pool
is loaded, so there is minimal performance overhead when doing operations
on the indirect vdev.
The size of the in-memory mapping table will be reduced when its entries
become "obsolete" because they are no longer used by any block pointers
in the pool. An entry becomes obsolete when all the blocks that use
it are freed. An entry can also become obsolete when all the snapshots
that reference it are deleted, and the block pointers that reference it
have been "remapped" in all filesystems/zvols (and clones). Whenever an
indirect block is written, all the block pointers in it will be "remapped"
to their new (concrete) locations if possible. This process can be
accelerated by using the "zfs remap" command to proactively rewrite all
indirect blocks that reference indirect (removed) vdevs.
Note that when a device is removed, we do not verify the checksum of
the data that is copied. This makes the process much faster, but if it
were used on redundant vdevs (i.e. mirror or raidz vdevs), it would be
possible to copy the wrong data, when we have the correct data on e.g.
the other side of the mirror.
At the moment, only mirrors and simple top-level vdevs can be removed
and no removal is allowed if any of the top-level vdevs are raidz.
Porting Notes:
* Avoid zero-sized kmem_alloc() in vdev_compact_children().
The device evacuation code adds a dependency that
vdev_compact_children() be able to properly empty the vdev_child
array by setting it to NULL and zeroing vdev_children. Under Linux,
kmem_alloc() and related functions return a sentinel pointer rather
than NULL for zero-sized allocations.
* Remove comment regarding "mpt" driver where zfs_remove_max_segment
is initialized to SPA_MAXBLOCKSIZE.
Change zfs_condense_indirect_commit_entry_delay_ticks to
zfs_condense_indirect_commit_entry_delay_ms for consistency with
most other tunables in which delays are specified in ms.
* ZTS changes:
Use set_tunable rather than mdb
Use zpool sync as appropriate
Use sync_pool instead of sync
Kill jobs during test_removal_with_operation to allow unmount/export
Don't add non-disk names such as "mirror" or "raidz" to $DISKS
Use $TEST_BASE_DIR instead of /tmp
Increase HZ from 100 to 1000 which is more common on Linux
removal_multiple_indirection.ksh
Reduce iterations in order to not time out on the code
coverage builders.
removal_resume_export:
Functionally, the test case is correct but there exists a race
where the kernel thread hasn't been fully started yet and is
not visible. Wait for up to 1 second for the removal thread
to be started before giving up on it. Also, increase the
amount of data copied in order that the removal not finish
before the export has a chance to fail.
* MMP compatibility, the concept of concrete versus non-concrete devices
has slightly changed the semantics of vdev_writeable(). Update
mmp_random_leaf_impl() accordingly.
* Updated dbuf_remap() to handle the org.zfsonlinux:large_dnode pool
feature which is not supported by OpenZFS.
* Added support for new vdev removal tracepoints.
* Test cases removal_with_zdb and removal_condense_export have been
intentionally disabled. When run manually they pass as intended,
but when running in the automated test environment they produce
unreliable results on the latest Fedora release.
They may work better once the upstream pool import refectoring is
merged into ZoL at which point they will be re-enabled.
Authored by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Alex Reece <alex@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed by: Richard Laager <rlaager@wiktel.com>
Reviewed by: Tim Chase <tim@chase2k.com>
Reviewed by: Brian Behlendorf <behlendorf1@llnl.gov>
Approved by: Garrett D'Amore <garrett@damore.org>
Ported-by: Tim Chase <tim@chase2k.com>
Signed-off-by: Tim Chase <tim@chase2k.com>
OpenZFS-issue: https://www.illumos.org/issues/7614
OpenZFS-commit: https://github.com/openzfs/openzfs/commit/f539f1eb
Closes #6900
2016-09-22 16:30:13 +00:00
|
|
|
if (dsl_dataset_remap_deadlist_exists(ds))
|
|
|
|
dsl_dataset_destroy_remap_deadlist(ds, tx);
|
|
|
|
|
2019-07-26 17:54:14 +00:00
|
|
|
/*
|
|
|
|
* Each destroy is responsible for both destroying (enqueuing
|
|
|
|
* to be destroyed) the blkptrs comprising the dataset as well as
|
|
|
|
* those belonging to the zil.
|
|
|
|
*/
|
|
|
|
if (dsl_deadlist_is_open(&ds->ds_dir->dd_livelist)) {
|
|
|
|
dsl_async_clone_destroy(ds, tx);
|
|
|
|
} else if (spa_feature_is_enabled(dp->dp_spa,
|
|
|
|
SPA_FEATURE_ASYNC_DESTROY)) {
|
|
|
|
dsl_async_dataset_destroy(ds, tx);
|
2013-09-04 12:00:57 +00:00
|
|
|
} else {
|
2019-07-26 17:54:14 +00:00
|
|
|
old_synchronous_dataset_destroy(ds, tx);
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (ds->ds_prev != NULL) {
|
|
|
|
if (spa_version(dp->dp_spa) >= SPA_VERSION_DIR_CLONES) {
|
|
|
|
VERIFY0(zap_remove_int(mos,
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dir_phys(ds->ds_prev->ds_dir)->dd_clones,
|
2013-09-04 12:00:57 +00:00
|
|
|
ds->ds_object, tx));
|
|
|
|
}
|
|
|
|
prevobj = ds->ds_prev->ds_object;
|
|
|
|
dsl_dataset_rele(ds->ds_prev, ds);
|
|
|
|
ds->ds_prev = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This must be done after the dsl_traverse(), because it will
|
|
|
|
* re-open the objset.
|
|
|
|
*/
|
|
|
|
if (ds->ds_objset) {
|
|
|
|
dmu_objset_evict(ds->ds_objset);
|
|
|
|
ds->ds_objset = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Erase the link in the dir */
|
|
|
|
dmu_buf_will_dirty(ds->ds_dir->dd_dbuf, tx);
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dir_phys(ds->ds_dir)->dd_head_dataset_obj = 0;
|
2013-09-04 12:00:57 +00:00
|
|
|
ddobj = ds->ds_dir->dd_object;
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT(dsl_dataset_phys(ds)->ds_snapnames_zapobj != 0);
|
|
|
|
VERIFY0(zap_destroy(mos,
|
|
|
|
dsl_dataset_phys(ds)->ds_snapnames_zapobj, tx));
|
2013-09-04 12:00:57 +00:00
|
|
|
|
Implement Redacted Send/Receive
Redacted send/receive allows users to send subsets of their data to
a target system. One possible use case for this feature is to not
transmit sensitive information to a data warehousing, test/dev, or
analytics environment. Another is to save space by not replicating
unimportant data within a given dataset, for example in backup tools
like zrepl.
Redacted send/receive is a three-stage process. First, a clone (or
clones) is made of the snapshot to be sent to the target. In this
clone (or clones), all unnecessary or unwanted data is removed or
modified. This clone is then snapshotted to create the "redaction
snapshot" (or snapshots). Second, the new zfs redact command is used
to create a redaction bookmark. The redaction bookmark stores the
list of blocks in a snapshot that were modified by the redaction
snapshot(s). Finally, the redaction bookmark is passed as a parameter
to zfs send. When sending to the snapshot that was redacted, the
redaction bookmark is used to filter out blocks that contain sensitive
or unwanted information, and those blocks are not included in the send
stream. When sending from the redaction bookmark, the blocks it
contains are considered as candidate blocks in addition to those
blocks in the destination snapshot that were modified since the
creation_txg of the redaction bookmark. This step is necessary to
allow the target to rehydrate data in the case where some blocks are
accidentally or unnecessarily modified in the redaction snapshot.
The changes to bookmarks to enable fast space estimation involve
adding deadlists to bookmarks. There is also logic to manage the
life cycles of these deadlists.
The new size estimation process operates in cases where previously
an accurate estimate could not be provided. In those cases, a send
is performed where no data blocks are read, reducing the runtime
significantly and providing a byte-accurate size estimate.
Reviewed-by: Dan Kimmel <dan.kimmel@delphix.com>
Reviewed-by: Matt Ahrens <mahrens@delphix.com>
Reviewed-by: Prashanth Sreenivasa <pks@delphix.com>
Reviewed-by: John Kennedy <john.kennedy@delphix.com>
Reviewed-by: George Wilson <george.wilson@delphix.com>
Reviewed-by: Chris Williamson <chris.williamson@delphix.com>
Reviewed-by: Pavel Zhakarov <pavel.zakharov@delphix.com>
Reviewed-by: Sebastien Roy <sebastien.roy@delphix.com>
Reviewed-by: Prakash Surya <prakash.surya@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Paul Dagnelie <pcd@delphix.com>
Closes #7958
2019-06-19 16:48:13 +00:00
|
|
|
if (ds->ds_bookmarks_obj != 0) {
|
|
|
|
void *cookie = NULL;
|
|
|
|
dsl_bookmark_node_t *dbn;
|
|
|
|
|
|
|
|
while ((dbn = avl_destroy_nodes(&ds->ds_bookmarks, &cookie)) !=
|
|
|
|
NULL) {
|
|
|
|
if (dbn->dbn_phys.zbm_redaction_obj != 0) {
|
|
|
|
VERIFY0(dmu_object_free(mos,
|
|
|
|
dbn->dbn_phys.zbm_redaction_obj, tx));
|
|
|
|
spa_feature_decr(dmu_objset_spa(mos),
|
|
|
|
SPA_FEATURE_REDACTION_BOOKMARKS, tx);
|
|
|
|
}
|
|
|
|
if (dbn->dbn_phys.zbm_flags & ZBM_FLAG_HAS_FBN) {
|
|
|
|
spa_feature_decr(dmu_objset_spa(mos),
|
|
|
|
SPA_FEATURE_BOOKMARK_WRITTEN, tx);
|
|
|
|
}
|
|
|
|
spa_strfree(dbn->dbn_name);
|
|
|
|
mutex_destroy(&dbn->dbn_lock);
|
|
|
|
kmem_free(dbn, sizeof (*dbn));
|
|
|
|
}
|
|
|
|
avl_destroy(&ds->ds_bookmarks);
|
|
|
|
VERIFY0(zap_destroy(mos, ds->ds_bookmarks_obj, tx));
|
2013-12-11 22:33:41 +00:00
|
|
|
spa_feature_decr(dp->dp_spa, SPA_FEATURE_BOOKMARKS, tx);
|
|
|
|
}
|
|
|
|
|
2013-09-04 12:00:57 +00:00
|
|
|
spa_prop_clear_bootfs(dp->dp_spa, ds->ds_object, tx);
|
|
|
|
|
2015-04-01 15:14:34 +00:00
|
|
|
ASSERT0(dsl_dataset_phys(ds)->ds_next_clones_obj);
|
|
|
|
ASSERT0(dsl_dataset_phys(ds)->ds_props_obj);
|
|
|
|
ASSERT0(dsl_dataset_phys(ds)->ds_userrefs_obj);
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_dir_rele(ds->ds_dir, ds);
|
|
|
|
ds->ds_dir = NULL;
|
2013-10-08 17:13:05 +00:00
|
|
|
dmu_object_free_zapified(mos, obj, tx);
|
2013-09-04 12:00:57 +00:00
|
|
|
|
|
|
|
dsl_dir_destroy_sync(ddobj, tx);
|
|
|
|
|
|
|
|
if (rmorigin) {
|
|
|
|
dsl_dataset_t *prev;
|
|
|
|
VERIFY0(dsl_dataset_hold_obj(dp, prevobj, FTAG, &prev));
|
|
|
|
dsl_destroy_snapshot_sync_impl(prev, B_FALSE, tx);
|
|
|
|
dsl_dataset_rele(prev, FTAG);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-02-08 16:16:23 +00:00
|
|
|
void
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_destroy_head_sync(void *arg, dmu_tx_t *tx)
|
|
|
|
{
|
|
|
|
dsl_destroy_head_arg_t *ddha = arg;
|
|
|
|
dsl_pool_t *dp = dmu_tx_pool(tx);
|
|
|
|
dsl_dataset_t *ds;
|
|
|
|
|
|
|
|
VERIFY0(dsl_dataset_hold(dp, ddha->ddha_name, FTAG, &ds));
|
|
|
|
dsl_destroy_head_sync_impl(ds, tx);
|
2014-03-22 09:07:14 +00:00
|
|
|
zvol_remove_minors(dp->dp_spa, ddha->ddha_name, B_TRUE);
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_dataset_rele(ds, FTAG);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
dsl_destroy_head_begin_sync(void *arg, dmu_tx_t *tx)
|
|
|
|
{
|
|
|
|
dsl_destroy_head_arg_t *ddha = arg;
|
|
|
|
dsl_pool_t *dp = dmu_tx_pool(tx);
|
|
|
|
dsl_dataset_t *ds;
|
|
|
|
|
|
|
|
VERIFY0(dsl_dataset_hold(dp, ddha->ddha_name, FTAG, &ds));
|
|
|
|
|
|
|
|
/* Mark it as inconsistent on-disk, in case we crash */
|
|
|
|
dmu_buf_will_dirty(ds->ds_dbuf, tx);
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_INCONSISTENT;
|
2013-09-04 12:00:57 +00:00
|
|
|
|
2019-09-12 20:28:26 +00:00
|
|
|
spa_history_log_internal_ds(ds, "destroy begin", tx, " ");
|
2013-09-04 12:00:57 +00:00
|
|
|
dsl_dataset_rele(ds, FTAG);
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
dsl_destroy_head(const char *name)
|
|
|
|
{
|
|
|
|
dsl_destroy_head_arg_t ddha;
|
|
|
|
int error;
|
|
|
|
spa_t *spa;
|
|
|
|
boolean_t isenabled;
|
|
|
|
|
|
|
|
#ifdef _KERNEL
|
|
|
|
zfs_destroy_unmount_origin(name);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
error = spa_open(name, &spa, FTAG);
|
|
|
|
if (error != 0)
|
|
|
|
return (error);
|
2013-10-08 17:13:05 +00:00
|
|
|
isenabled = spa_feature_is_enabled(spa, SPA_FEATURE_ASYNC_DESTROY);
|
2013-09-04 12:00:57 +00:00
|
|
|
spa_close(spa, FTAG);
|
|
|
|
|
|
|
|
ddha.ddha_name = name;
|
|
|
|
|
|
|
|
if (!isenabled) {
|
|
|
|
objset_t *os;
|
|
|
|
|
|
|
|
error = dsl_sync_task(name, dsl_destroy_head_check,
|
2014-11-03 20:28:43 +00:00
|
|
|
dsl_destroy_head_begin_sync, &ddha,
|
2016-12-16 22:11:29 +00:00
|
|
|
0, ZFS_SPACE_CHECK_DESTROY);
|
2013-09-04 12:00:57 +00:00
|
|
|
if (error != 0)
|
|
|
|
return (error);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Head deletion is processed in one txg on old pools;
|
|
|
|
* remove the objects from open context so that the txg sync
|
2019-07-15 23:08:42 +00:00
|
|
|
* is not too long. This optimization can only work for
|
|
|
|
* encrypted datasets if the wrapping key is loaded.
|
2013-09-04 12:00:57 +00:00
|
|
|
*/
|
2019-07-15 23:08:42 +00:00
|
|
|
error = dmu_objset_own(name, DMU_OST_ANY, B_FALSE, B_TRUE,
|
Native Encryption for ZFS on Linux
This change incorporates three major pieces:
The first change is a keystore that manages wrapping
and encryption keys for encrypted datasets. These
commands mostly involve manipulating the new
DSL Crypto Key ZAP Objects that live in the MOS. Each
encrypted dataset has its own DSL Crypto Key that is
protected with a user's key. This level of indirection
allows users to change their keys without re-encrypting
their entire datasets. The change implements the new
subcommands "zfs load-key", "zfs unload-key" and
"zfs change-key" which allow the user to manage their
encryption keys and settings. In addition, several new
flags and properties have been added to allow dataset
creation and to make mounting and unmounting more
convenient.
The second piece of this patch provides the ability to
encrypt, decyrpt, and authenticate protected datasets.
Each object set maintains a Merkel tree of Message
Authentication Codes that protect the lower layers,
similarly to how checksums are maintained. This part
impacts the zio layer, which handles the actual
encryption and generation of MACs, as well as the ARC
and DMU, which need to be able to handle encrypted
buffers and protected data.
The last addition is the ability to do raw, encrypted
sends and receives. The idea here is to send raw
encrypted and compressed data and receive it exactly
as is on a backup system. This means that the dataset
on the receiving system is protected using the same
user key that is in use on the sending side. By doing
so, datasets can be efficiently backed up to an
untrusted system without fear of data being
compromised.
Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: Jorgen Lundman <lundman@lundman.net>
Signed-off-by: Tom Caputi <tcaputi@datto.com>
Closes #494
Closes #5769
2017-08-14 17:36:48 +00:00
|
|
|
FTAG, &os);
|
2013-09-04 12:00:57 +00:00
|
|
|
if (error == 0) {
|
|
|
|
uint64_t prev_snap_txg =
|
2015-04-01 15:14:34 +00:00
|
|
|
dsl_dataset_phys(dmu_objset_ds(os))->
|
|
|
|
ds_prev_snap_txg;
|
2017-11-04 20:25:13 +00:00
|
|
|
for (uint64_t obj = 0; error == 0;
|
2013-09-04 12:00:57 +00:00
|
|
|
error = dmu_object_next(os, &obj, FALSE,
|
|
|
|
prev_snap_txg))
|
2013-08-21 04:11:52 +00:00
|
|
|
(void) dmu_free_long_object(os, obj);
|
2013-09-04 12:00:57 +00:00
|
|
|
/* sync out all frees */
|
|
|
|
txg_wait_synced(dmu_objset_pool(os), 0);
|
2019-07-15 23:08:42 +00:00
|
|
|
dmu_objset_disown(os, B_TRUE, FTAG);
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return (dsl_sync_task(name, dsl_destroy_head_check,
|
2016-12-16 22:11:29 +00:00
|
|
|
dsl_destroy_head_sync, &ddha, 0, ZFS_SPACE_CHECK_DESTROY));
|
2013-09-04 12:00:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Note, this function is used as the callback for dmu_objset_find(). We
|
|
|
|
* always return 0 so that we will continue to find and process
|
|
|
|
* inconsistent datasets, even if we encounter an error trying to
|
|
|
|
* process one of them.
|
|
|
|
*/
|
|
|
|
/* ARGSUSED */
|
|
|
|
int
|
|
|
|
dsl_destroy_inconsistent(const char *dsname, void *arg)
|
|
|
|
{
|
|
|
|
objset_t *os;
|
|
|
|
|
|
|
|
if (dmu_objset_hold(dsname, FTAG, &os) == 0) {
|
2016-01-06 21:22:48 +00:00
|
|
|
boolean_t need_destroy = DS_IS_INCONSISTENT(dmu_objset_ds(os));
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the dataset is inconsistent because a resumable receive
|
|
|
|
* has failed, then do not destroy it.
|
|
|
|
*/
|
|
|
|
if (dsl_dataset_has_resume_receive_state(dmu_objset_ds(os)))
|
|
|
|
need_destroy = B_FALSE;
|
|
|
|
|
2013-09-04 12:00:57 +00:00
|
|
|
dmu_objset_rele(os, FTAG);
|
2016-01-06 21:22:48 +00:00
|
|
|
if (need_destroy)
|
2013-09-04 12:00:57 +00:00
|
|
|
(void) dsl_destroy_head(dsname);
|
|
|
|
}
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2018-02-16 01:53:18 +00:00
|
|
|
#if defined(_KERNEL)
|
2013-09-04 12:00:57 +00:00
|
|
|
EXPORT_SYMBOL(dsl_destroy_head);
|
|
|
|
EXPORT_SYMBOL(dsl_destroy_head_sync_impl);
|
|
|
|
EXPORT_SYMBOL(dsl_dataset_user_hold_check_one);
|
|
|
|
EXPORT_SYMBOL(dsl_destroy_snapshot_sync_impl);
|
|
|
|
EXPORT_SYMBOL(dsl_destroy_inconsistent);
|
|
|
|
EXPORT_SYMBOL(dsl_dataset_user_release_tmp);
|
|
|
|
EXPORT_SYMBOL(dsl_destroy_head_check_impl);
|
|
|
|
#endif
|