d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove deduplicated send/receive code

Browse Source 
Deduplicated send streams (i.e. `zfs send -D` and `zfs receive` of such
streams) are deprecated.  Deduplicated send streams can be received by
first converting them to non-deduplicated with the `zstream redup`
command.

This commit removes the code for sending and receiving deduplicated send
streams.  `zfs send -D` will now print a warning, ignore the `-D` flag,
and generate a regular (non-deduplicated) send stream.  `zfs receive` of
a deduplicated send stream will print an error message and fail.

The resulting code simplification (especially in the kernel's support
for receiving dedup streams) should help enable future performance
enhancements.

Several new tests are added which leverage `zstream redup`.

Reviewed-by: Paul Dagnelie <pcd@delphix.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Matthew Ahrens <mahrens@delphix.com>
Issue #7887
Issue #10117
Issue #10156
Closes #10212
This commit is contained in:
Matthew Ahrens 2020-04-23 10:06:57 -07:00 committed by GitHub
parent 70e5ad31f6
commit 196bee4cfd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
26 changed files with 219 additions and 1081 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
cmd/zfs/zfs_main.c
View File

@ -21,7 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2019 by Delphix. All rights reserved.
* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
* Copyright 2012 Milan Jurik. All rights reserved.
* Copyright (c) 2012, Joyent, Inc. All rights reserved.
* Copyright (c) 2013 Steven Hartland. All rights reserved.
@ -4266,7 +4266,10 @@ zfs_do_send(int argc, char **argv)
flags.progress = B_TRUE;
break;
case 'D':
flags.dedup = B_TRUE;
(void) fprintf(stderr,
gettext("WARNING: deduplicated send is no "
"longer supported. A regular,\n"
"non-deduplicated stream will be generated.\n\n"));
break;
case 'n':
flags.dryrun = B_TRUE;
@ -4333,16 +4336,6 @@ zfs_do_send(int argc, char **argv)
}
}
if (flags.dedup) {
(void) fprintf(stderr,
gettext("WARNING: deduplicated send is "
"deprecated, and will be removed in a\n"
"future release. (In the future, the flag will be "
"accepted, but a\n"
"regular, non-deduplicated stream will be "
"generated.)\n\n"));
}
if (flags.parsable && flags.verbosity == 0)
flags.verbosity = 1;
@ -4351,7 +4344,7 @@ zfs_do_send(int argc, char **argv)
if (resume_token != NULL) {
if (fromname != NULL || flags.replicate || flags.props ||
flags.backup || flags.dedup || flags.holds ||
flags.backup || flags.holds ||
flags.saved || redactbook != NULL) {
(void) fprintf(stderr,
gettext("invalid flags combined with -t\n"));
@ -4375,7 +4368,7 @@ zfs_do_send(int argc, char **argv)
if (flags.saved) {
if (fromname != NULL || flags.replicate || flags.props ||
flags.doall || flags.backup || flags.dedup ||
flags.doall || flags.backup ||
flags.holds || flags.largeblock || flags.embed_data ||
flags.compress || flags.raw || redactbook != NULL) {
(void) fprintf(stderr, gettext("incompatible flags "

6
include/libzfs.h
View File

@ -21,7 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2018 by Delphix. All rights reserved.
* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
* Copyright Joyent, Inc.
* Copyright (c) 2013 Steven Hartland. All rights reserved.
* Copyright (c) 2016, Intel Corporation.
@ -651,8 +651,8 @@ typedef struct sendflags {
/* if dataset is a clone, do incremental from its origin */
boolean_t fromorigin;
/* do deduplication */
boolean_t dedup;
/* field no longer used, maintained for backwards compatibility */
boolean_t pad;
/* send properties (ie, -p) */
boolean_t props;

3
include/libzfs_impl.h
View File

@ -21,7 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2017 by Delphix. All rights reserved.
* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
* Copyright (c) 2018 Datto Inc.
*/
@ -71,7 +71,6 @@ struct libzfs_handle {
int libzfs_pool_iter;
char libzfs_chassis_id[256];
boolean_t libzfs_prop_debug;
boolean_t libzfs_dedup_warning_printed;
};
#define ZFSSHARE_MISS 0x01 /* Didn't find entry in cache */

4
include/sys/dmu.h
View File

@ -20,7 +20,7 @@
*/
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2018 by Delphix. All rights reserved.
* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
* Copyright (c) 2012, Joyent, Inc. All rights reserved.
* Copyright 2014 HybridCluster. All rights reserved.
@ -864,8 +864,6 @@ int dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset,
int dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset,
struct arc_buf *buf, dmu_tx_t *tx);
#define dmu_assign_arcbuf dmu_assign_arcbuf_by_dbuf
void dmu_copy_from_buf(objset_t *os, uint64_t object, uint64_t offset,
dmu_buf_t *handle, dmu_tx_t *tx);
#ifdef HAVE_UIO_ZEROCOPY
int dmu_xuio_init(struct xuio *uio, int niov);
void dmu_xuio_fini(struct xuio *uio);

18
include/sys/dmu_recv.h
View File

@ -21,7 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
* Copyright (c) 2012, 2020 by Delphix. All rights reserved.
* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
* Copyright (c) 2013, Joyent, Inc. All rights reserved.
*/
@ -53,10 +53,8 @@ typedef struct dmu_recv_cookie {
boolean_t drc_raw;
boolean_t drc_clone;
boolean_t drc_spill;
struct avl_tree *drc_guid_to_ds_map;
nvlist_t *drc_keynvl;
uint64_t drc_fromsnapobj;
uint64_t drc_newsnapobj;
uint64_t drc_ivset_guid;
void *drc_owner;
cred_t *drc_cred;
@ -80,13 +78,11 @@ typedef struct dmu_recv_cookie {
objlist_t *drc_ignore_objlist;
} dmu_recv_cookie_t;
int dmu_recv_begin(char *tofs, char *tosnap, dmu_replay_record_t *drr_begin,
boolean_t force, boolean_t resumable, nvlist_t *localprops,
nvlist_t *hidden_args, char *origin, dmu_recv_cookie_t *drc,
zfs_file_t *fp, offset_t *voffp);
int dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd,
uint64_t *action_handlep, offset_t *voffp);
int dmu_recv_end(dmu_recv_cookie_t *drc, void *owner);
boolean_t dmu_objset_is_receiving(objset_t *os);
int dmu_recv_begin(char *, char *, dmu_replay_record_t *,
boolean_t, boolean_t, nvlist_t *, nvlist_t *, char *,
dmu_recv_cookie_t *, zfs_file_t *, offset_t *);
int dmu_recv_stream(dmu_recv_cookie_t *, offset_t *);
int dmu_recv_end(dmu_recv_cookie_t *, void *);
boolean_t dmu_objset_is_receiving(objset_t *);
#endif /* _DMU_RECV_H */

5
include/sys/zfs_ioctl.h
View File

@ -20,7 +20,7 @@
*/
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
* Copyright (c) 2012, 2020 by Delphix. All rights reserved.
* Copyright 2016 RackTop Systems.
* Copyright (c) 2017, Intel Corporation.
*/
@ -111,8 +111,7 @@ typedef enum drr_headertype {
/*
* Mask of all supported backup features
*/
#define DMU_BACKUP_FEATURE_MASK (DMU_BACKUP_FEATURE_DEDUP | \
DMU_BACKUP_FEATURE_DEDUPPROPS | DMU_BACKUP_FEATURE_SA_SPILL | \
#define DMU_BACKUP_FEATURE_MASK (DMU_BACKUP_FEATURE_SA_SPILL | \
DMU_BACKUP_FEATURE_EMBED_DATA | DMU_BACKUP_FEATURE_LZ4 | \
DMU_BACKUP_FEATURE_RESUMING | DMU_BACKUP_FEATURE_LARGE_BLOCKS | \
DMU_BACKUP_FEATURE_COMPRESSED | DMU_BACKUP_FEATURE_LARGE_DNODE | \

5
include/sys/zfs_onexit.h
View File

@ -21,6 +21,7 @@
/*
* Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2020 by Delphix. All rights reserved.
*/
#ifndef _SYS_ZFS_ONEXIT_H
@ -54,10 +55,6 @@ extern int zfs_onexit_fd_hold(int fd, minor_t *minorp);
extern void zfs_onexit_fd_rele(int fd);
extern int zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
uint64_t *action_handle);
extern int zfs_onexit_del_cb(minor_t minor, uint64_t action_handle,
boolean_t fire);
extern int zfs_onexit_cb_data(minor_t minor, uint64_t action_handle,
void **data);
#ifdef __cplusplus
}

523
lib/libzfs/libzfs_sendrecv.c
View File

@ -21,7 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2018 by Delphix. All rights reserved.
* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
* Copyright (c) 2012, Joyent, Inc. All rights reserved.
* Copyright (c) 2012 Pawel Jakub Dawidek <pawel@dawidek.net>.
* All rights reserved
@ -73,22 +73,14 @@
extern void zfs_setprop_error(libzfs_handle_t *, zfs_prop_t, int, char *);
static int zfs_receive_impl(libzfs_handle_t *, const char *, const char *,
recvflags_t *, int, const char *, nvlist_t *, avl_tree_t *, char **, int,
uint64_t *, const char *, nvlist_t *);
recvflags_t *, int, const char *, nvlist_t *, avl_tree_t *, char **,
const char *, nvlist_t *);
static int guid_to_name_redact_snaps(libzfs_handle_t *hdl, const char *parent,
uint64_t guid, boolean_t bookmark_ok, uint64_t *redact_snap_guids,
uint64_t num_redact_snaps, char *name);
static int guid_to_name(libzfs_handle_t *, const char *,
uint64_t, boolean_t, char *);
static const zio_cksum_t zero_cksum = { { 0 } };
typedef struct dedup_arg {
int inputfd;
int outputfd;
libzfs_handle_t *dedup_hdl;
} dedup_arg_t;
typedef struct progress_arg {
zfs_handle_t *pa_zhp;
int pa_fd;
@ -97,112 +89,6 @@ typedef struct progress_arg {
int pa_verbosity;
} progress_arg_t;
typedef struct dataref {
uint64_t ref_guid;
uint64_t ref_object;
uint64_t ref_offset;
} dataref_t;
typedef struct dedup_entry {
struct dedup_entry *dde_next;
zio_cksum_t dde_chksum;
uint64_t dde_prop;
dataref_t dde_ref;
} dedup_entry_t;
#define MAX_DDT_PHYSMEM_PERCENT 20
#define SMALLEST_POSSIBLE_MAX_DDT_MB 128
typedef struct dedup_table {
dedup_entry_t **dedup_hash_array;
umem_cache_t *ddecache;
uint64_t max_ddt_size; /* max dedup table size in bytes */
uint64_t cur_ddt_size; /* current dedup table size in bytes */
uint64_t ddt_count;
int numhashbits;
boolean_t ddt_full;
} dedup_table_t;
static int
high_order_bit(uint64_t n)
{
int count;
for (count = 0; n != 0; count++)
n >>= 1;
return (count);
}
static size_t
ssread(void *buf, size_t len, FILE *stream)
{
size_t outlen;
if ((outlen = fread(buf, len, 1, stream)) == 0)
return (0);
return (outlen);
}
static void
ddt_hash_append(libzfs_handle_t *hdl, dedup_table_t *ddt, dedup_entry_t **ddepp,
zio_cksum_t *cs, uint64_t prop, dataref_t *dr)
{
dedup_entry_t *dde;
if (ddt->cur_ddt_size >= ddt->max_ddt_size) {
if (ddt->ddt_full == B_FALSE) {
zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
"Dedup table full. Deduplication will continue "
"with existing table entries"));
ddt->ddt_full = B_TRUE;
}
return;
}
if ((dde = umem_cache_alloc(ddt->ddecache, UMEM_DEFAULT))
!= NULL) {
assert(*ddepp == NULL);
dde->dde_next = NULL;
dde->dde_chksum = *cs;
dde->dde_prop = prop;
dde->dde_ref = *dr;
*ddepp = dde;
ddt->cur_ddt_size += sizeof (dedup_entry_t);
ddt->ddt_count++;
}
}
/*
* Using the specified dedup table, do a lookup for an entry with
* the checksum cs. If found, return the block's reference info
* in *dr. Otherwise, insert a new entry in the dedup table, using
* the reference information specified by *dr.
*
* return value: true - entry was found
* false - entry was not found
*/
static boolean_t
ddt_update(libzfs_handle_t *hdl, dedup_table_t *ddt, zio_cksum_t *cs,
uint64_t prop, dataref_t *dr)
{
uint32_t hashcode;
dedup_entry_t **ddepp;
hashcode = BF64_GET(cs->zc_word[0], 0, ddt->numhashbits);
for (ddepp = &(ddt->dedup_hash_array[hashcode]); *ddepp != NULL;
ddepp = &((*ddepp)->dde_next)) {
if (ZIO_CHECKSUM_EQUAL(((*ddepp)->dde_chksum), *cs) &&
(*ddepp)->dde_prop == prop) {
*dr = (*ddepp)->dde_ref;
return (B_TRUE);
}
}
ddt_hash_append(hdl, ddt, ddepp, cs, prop, dr);
return (B_FALSE);
}
static int
dump_record(dmu_replay_record_t *drr, void *payload, int payload_len,
zio_cksum_t *zc, int outfd)
@ -228,274 +114,6 @@ dump_record(dmu_replay_record_t *drr, void *payload, int payload_len,
return (0);
}
/*
* This function is started in a separate thread when the dedup option
* has been requested. The main send thread determines the list of
* snapshots to be included in the send stream and makes the ioctl calls
* for each one. But instead of having the ioctl send the output to the
* the output fd specified by the caller of zfs_send()), the
* ioctl is told to direct the output to a pipe, which is read by the
* alternate thread running THIS function. This function does the
* dedup'ing by:
* 1. building a dedup table (the DDT)
* 2. doing checksums on each data block and inserting a record in the DDT
* 3. looking for matching checksums, and
* 4. sending a DRR_WRITE_BYREF record instead of a write record whenever
* a duplicate block is found.
* The output of this function then goes to the output fd requested
* by the caller of zfs_send().
*/
static void *
cksummer(void *arg)
{
dedup_arg_t *dda = arg;
char *buf = zfs_alloc(dda->dedup_hdl, SPA_MAXBLOCKSIZE);
dmu_replay_record_t thedrr = { 0 };
dmu_replay_record_t *drr = &thedrr;
FILE *ofp;
int outfd;
dedup_table_t ddt;
zio_cksum_t stream_cksum;
uint64_t numbuckets;
#ifdef _ILP32
ddt.max_ddt_size = SMALLEST_POSSIBLE_MAX_DDT_MB << 20;
#else
uint64_t physmem = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
ddt.max_ddt_size =
MAX((physmem * MAX_DDT_PHYSMEM_PERCENT) / 100,
SMALLEST_POSSIBLE_MAX_DDT_MB << 20);
#endif
numbuckets = ddt.max_ddt_size / (sizeof (dedup_entry_t));
/*
* numbuckets must be a power of 2. Increase number to
* a power of 2 if necessary.
*/
if (!ISP2(numbuckets))
numbuckets = 1ULL << high_order_bit(numbuckets);
ddt.dedup_hash_array = calloc(numbuckets, sizeof (dedup_entry_t *));
ddt.ddecache = umem_cache_create("dde", sizeof (dedup_entry_t), 0,
NULL, NULL, NULL, NULL, NULL, 0);
ddt.cur_ddt_size = numbuckets * sizeof (dedup_entry_t *);
ddt.numhashbits = high_order_bit(numbuckets) - 1;
ddt.ddt_full = B_FALSE;
outfd = dda->outputfd;
ofp = fdopen(dda->inputfd, "r");
while (ssread(drr, sizeof (*drr), ofp) != 0) {
/*
* kernel filled in checksum, we are going to write same
* record, but need to regenerate checksum.
*/
if (drr->drr_type != DRR_BEGIN) {
bzero(&drr->drr_u.drr_checksum.drr_checksum,
sizeof (drr->drr_u.drr_checksum.drr_checksum));
}
switch (drr->drr_type) {
case DRR_BEGIN:
{
struct drr_begin *drrb = &drr->drr_u.drr_begin;
int fflags;
int sz = 0;
ZIO_SET_CHECKSUM(&stream_cksum, 0, 0, 0, 0);
ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC);
/* set the DEDUP feature flag for this stream */
fflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
fflags |= (DMU_BACKUP_FEATURE_DEDUP |
DMU_BACKUP_FEATURE_DEDUPPROPS);
DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, fflags);
if (drr->drr_payloadlen != 0) {
sz = drr->drr_payloadlen;
if (sz > SPA_MAXBLOCKSIZE) {
buf = zfs_realloc(dda->dedup_hdl, buf,
SPA_MAXBLOCKSIZE, sz);
}
(void) ssread(buf, sz, ofp);
if (ferror(stdin))
perror("fread");
}
if (dump_record(drr, buf, sz, &stream_cksum,
outfd) != 0)
goto out;
break;
}
case DRR_END:
{
struct drr_end *drre = &drr->drr_u.drr_end;
/* use the recalculated checksum */
drre->drr_checksum = stream_cksum;
if (dump_record(drr, NULL, 0, &stream_cksum,
outfd) != 0)
goto out;
break;
}
case DRR_OBJECT:
{
struct drr_object *drro = &drr->drr_u.drr_object;
if (drro->drr_bonuslen > 0) {
(void) ssread(buf,
DRR_OBJECT_PAYLOAD_SIZE(drro), ofp);
}
if (dump_record(drr, buf, DRR_OBJECT_PAYLOAD_SIZE(drro),
&stream_cksum, outfd) != 0)
goto out;
break;
}
case DRR_SPILL:
{
struct drr_spill *drrs = &drr->drr_u.drr_spill;
(void) ssread(buf, DRR_SPILL_PAYLOAD_SIZE(drrs), ofp);
if (dump_record(drr, buf, DRR_SPILL_PAYLOAD_SIZE(drrs),
&stream_cksum, outfd) != 0)
goto out;
break;
}
case DRR_FREEOBJECTS:
{
if (dump_record(drr, NULL, 0, &stream_cksum,
outfd) != 0)
goto out;
break;
}
case DRR_WRITE:
{
struct drr_write *drrw = &drr->drr_u.drr_write;
dataref_t dataref;
uint64_t payload_size;
payload_size = DRR_WRITE_PAYLOAD_SIZE(drrw);
(void) ssread(buf, payload_size, ofp);
/*
* Use the existing checksum if it's dedup-capable,
* else calculate a SHA256 checksum for it.
*/
if (ZIO_CHECKSUM_EQUAL(drrw->drr_key.ddk_cksum,
zero_cksum) ||
!DRR_IS_DEDUP_CAPABLE(drrw->drr_flags)) {
SHA2_CTX ctx;
zio_cksum_t tmpsha256;
SHA2Init(SHA256, &ctx);
SHA2Update(&ctx, buf, payload_size);
SHA2Final(&tmpsha256, &ctx);
drrw->drr_key.ddk_cksum.zc_word[0] =
BE_64(tmpsha256.zc_word[0]);
drrw->drr_key.ddk_cksum.zc_word[1] =
BE_64(tmpsha256.zc_word[1]);
drrw->drr_key.ddk_cksum.zc_word[2] =
BE_64(tmpsha256.zc_word[2]);
drrw->drr_key.ddk_cksum.zc_word[3] =
BE_64(tmpsha256.zc_word[3]);
drrw->drr_checksumtype = ZIO_CHECKSUM_SHA256;
drrw->drr_flags |= DRR_CHECKSUM_DEDUP;
}
dataref.ref_guid = drrw->drr_toguid;
dataref.ref_object = drrw->drr_object;
dataref.ref_offset = drrw->drr_offset;
if (ddt_update(dda->dedup_hdl, &ddt,
&drrw->drr_key.ddk_cksum, drrw->drr_key.ddk_prop,
&dataref)) {
dmu_replay_record_t wbr_drr = {0};
struct drr_write_byref *wbr_drrr =
&wbr_drr.drr_u.drr_write_byref;
/* block already present in stream */
wbr_drr.drr_type = DRR_WRITE_BYREF;
wbr_drrr->drr_object = drrw->drr_object;
wbr_drrr->drr_offset = drrw->drr_offset;
wbr_drrr->drr_length = drrw->drr_logical_size;
wbr_drrr->drr_toguid = drrw->drr_toguid;
wbr_drrr->drr_refguid = dataref.ref_guid;
wbr_drrr->drr_refobject =
dataref.ref_object;
wbr_drrr->drr_refoffset =
dataref.ref_offset;
wbr_drrr->drr_checksumtype =
drrw->drr_checksumtype;
wbr_drrr->drr_flags = drrw->drr_flags;
wbr_drrr->drr_key.ddk_cksum =
drrw->drr_key.ddk_cksum;
wbr_drrr->drr_key.ddk_prop =
drrw->drr_key.ddk_prop;
if (dump_record(&wbr_drr, NULL, 0,
&stream_cksum, outfd) != 0)
goto out;
} else {
/* block not previously seen */
if (dump_record(drr, buf, payload_size,
&stream_cksum, outfd) != 0)
goto out;
}
break;
}
case DRR_WRITE_EMBEDDED:
{
struct drr_write_embedded *drrwe =
&drr->drr_u.drr_write_embedded;
(void) ssread(buf,
P2ROUNDUP((uint64_t)drrwe->drr_psize, 8), ofp);
if (dump_record(drr, buf,
P2ROUNDUP((uint64_t)drrwe->drr_psize, 8),
&stream_cksum, outfd) != 0)
goto out;
break;
}
case DRR_FREE:
{
if (dump_record(drr, NULL, 0, &stream_cksum,
outfd) != 0)
goto out;
break;
}
case DRR_OBJECT_RANGE:
{
if (dump_record(drr, NULL, 0, &stream_cksum,
outfd) != 0)
goto out;
break;
}
default:
(void) fprintf(stderr, "INVALID record type 0x%x\n",
drr->drr_type);
/* should never happen, so assert */
assert(B_FALSE);
}
}
out:
umem_cache_destroy(ddt.ddecache);
free(ddt.dedup_hash_array);
free(buf);
(void) fclose(ofp);
return (NULL);
}
/*
* Routines for dealing with the AVL tree of fs-nvlists
*/
@ -2478,7 +2096,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
int spa_version;
pthread_t tid = 0;
int pipefd[2];
dedup_arg_t dda = { 0 };
int featureflags = 0;
FILE *fout;
@ -2502,33 +2119,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
if (flags->holds)
featureflags |= DMU_BACKUP_FEATURE_HOLDS;
/*
* Start the dedup thread if this is a dedup stream. We do not bother
* doing this if this a raw send of an encrypted dataset with dedup off
* because normal encrypted blocks won't dedup.
*/
if (flags->dedup && !flags->dryrun && !(flags->raw &&
zfs_prop_get_int(zhp, ZFS_PROP_ENCRYPTION) != ZIO_CRYPT_OFF &&
zfs_prop_get_int(zhp, ZFS_PROP_DEDUP) == ZIO_CHECKSUM_OFF)) {
featureflags |= (DMU_BACKUP_FEATURE_DEDUP |
DMU_BACKUP_FEATURE_DEDUPPROPS);
if ((err = socketpair(AF_UNIX, SOCK_STREAM, 0, pipefd)) != 0) {
zfs_error_aux(zhp->zfs_hdl, strerror(errno));
return (zfs_error(zhp->zfs_hdl, EZFS_PIPEFAILED,
errbuf));
}
dda.outputfd = outfd;
dda.inputfd = pipefd[1];
dda.dedup_hdl = zhp->zfs_hdl;
if ((err = pthread_create(&tid, NULL, cksummer, &dda)) != 0) {
(void) close(pipefd[0]);
(void) close(pipefd[1]);
zfs_error_aux(zhp->zfs_hdl, strerror(errno));
return (zfs_error(zhp->zfs_hdl,
EZFS_THREADCREATEFAILED, errbuf));
}
}
if (flags->replicate || flags->doall || flags->props ||
flags->holds || flags->backup) {
char full_tosnap_name[ZFS_MAX_DATASET_NAME_LEN];
@ -2706,34 +2296,6 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap,
return (err);
}
static int
get_dedup_fd(zfs_handle_t *zhp, dedup_arg_t *dda, int fd, pthread_t *tid,
int *outfd)
{
int pipefd[2];
char errbuf[1024];
int err;
(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
"warning: cannot send '%s'"), zhp->zfs_name);
if ((err = socketpair(AF_UNIX, SOCK_STREAM, 0, pipefd)) != 0) {
zfs_error_aux(zhp->zfs_hdl, strerror(errno));
return (zfs_error(zhp->zfs_hdl, EZFS_PIPEFAILED,
errbuf));
}
dda->outputfd = fd;
dda->inputfd = pipefd[1];
dda->dedup_hdl = zhp->zfs_hdl;
if ((err = pthread_create(tid, NULL, cksummer, dda)) != 0) {
(void) close(pipefd[0]);
(void) close(pipefd[1]);
zfs_error_aux(zhp->zfs_hdl, strerror(err));
return (zfs_error(zhp->zfs_hdl, EZFS_THREADCREATEFAILED,
errbuf));
}
*outfd = pipefd[0];
return (0);
}
zfs_handle_t *
name_to_dir_handle(libzfs_handle_t *hdl, const char *snapname)
{
@ -2819,9 +2381,8 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
libzfs_handle_t *hdl = zhp->zfs_hdl;
char *name = zhp->zfs_name;
int orig_fd = fd;
pthread_t ddtid, ptid;
pthread_t ptid;
progress_arg_t pa = { 0 };
dedup_arg_t dda = { 0 };
char errbuf[1024];
(void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN,
@ -2914,16 +2475,6 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
if (flags->dryrun)
return (0);
/*
* If deduplication is requested, spawn a thread that will deduplicate
* the data coming out of the kernel.
*/
if (flags->dedup) {
err = get_dedup_fd(zhp, &dda, fd, &ddtid, &fd);
if (err != 0)
return (err);
}
/*
* If progress reporting is requested, spawn a new thread to poll
* ZFS_IOC_SEND_PROGRESS at a regular interval.
@ -2939,11 +2490,6 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
send_progress_thread, &pa);
if (err != 0) {
zfs_error_aux(zhp->zfs_hdl, strerror(errno));
if (flags->dedup) {
(void) pthread_cancel(ddtid);
(void) close(fd);
(void) pthread_join(ddtid, NULL);
}
return (zfs_error(zhp->zfs_hdl,
EZFS_THREADCREATEFAILED, errbuf));
}
@ -2966,12 +2512,6 @@ zfs_send_one(zfs_handle_t *zhp, const char *from, int fd, sendflags_t *flags,
return (zfs_standard_error(hdl, error, errbuf));
}
}
if (flags->dedup) {
if (err != 0)
(void) pthread_cancel(ddtid);
(void) close(fd);
(void) pthread_join(ddtid, NULL);
}
if (flags->props || flags->holds || flags->backup) {
/* Write the final end record. */
@ -3965,8 +3505,7 @@ recv_incremental_replication(libzfs_handle_t *hdl, const char *tofs,
static int
zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname,
recvflags_t *flags, dmu_replay_record_t *drr, zio_cksum_t *zc,
char **top_zfs, int cleanup_fd, uint64_t *action_handlep,
nvlist_t *cmdprops)
char **top_zfs, nvlist_t *cmdprops)
{
nvlist_t *stream_nv = NULL;
avl_tree_t *stream_avl = NULL;
@ -4143,8 +3682,7 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname,
* recv_skip() and return 0).
*/
error = zfs_receive_impl(hdl, destname, NULL, flags, fd,
sendfs, stream_nv, stream_avl, top_zfs, cleanup_fd,
action_handlep, sendsnap, cmdprops);
sendfs, stream_nv, stream_avl, top_zfs, sendsnap, cmdprops);
if (error == ENODATA) {
error = 0;
break;
@ -4530,8 +4068,8 @@ static int
zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
const char *originsnap, recvflags_t *flags, dmu_replay_record_t *drr,
dmu_replay_record_t *drr_noswap, const char *sendfs, nvlist_t *stream_nv,
avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd,
uint64_t *action_handlep, const char *finalsnap, nvlist_t *cmdprops)
avl_tree_t *stream_avl, char **top_zfs,
const char *finalsnap, nvlist_t *cmdprops)
{
time_t begin_time;
int ioctl_err, ioctl_errno, err;
@ -4741,24 +4279,16 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
(void) printf("found clone origin %s\n", origin);
}
if (!hdl->libzfs_dedup_warning_printed &&
(DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
if ((DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
DMU_BACKUP_FEATURE_DEDUP)) {
(void) fprintf(stderr,
gettext("WARNING: This is a deduplicated send stream. "
"The ability to send and\n"
"receive deduplicated send streams is deprecated. "
"In the future, the\n"
"ability to receive a deduplicated send stream with "
"\"zfs receive\" will be\n"
"removed. However, in the future, a utility will be "
"provided to convert a\n"
"deduplicated send stream to a regular "
"(non-deduplicated) stream. This\n"
"future utility will require that the send stream be "
"located in a\n"
"seek-able file, rather than provided by a pipe.\n\n"));
hdl->libzfs_dedup_warning_printed = B_TRUE;
gettext("ERROR: \"zfs receive\" no longer supports "
"deduplicated send streams. Use\n"
"the \"zstream redup\" command to convert this stream "
"to a regular,\n"
"non-deduplicated stream.\n"));
err = zfs_error(hdl, EZFS_NOTSUP, errbuf);
goto out;
}
boolean_t resuming = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) &
@ -5103,8 +4633,8 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap,
err = ioctl_err = lzc_receive_with_cmdprops(destsnap, rcvprops,
oxprops, wkeydata, wkeylen, origin, flags->force, flags->resumable,
raw, infd, drr_noswap, cleanup_fd, &read_bytes, &errflags,
action_handlep, &prop_errors);
raw, infd, drr_noswap, -1, &read_bytes, &errflags,
NULL, &prop_errors);
ioctl_errno = ioctl_err;
prop_errflags = errflags;
@ -5435,8 +4965,8 @@ zfs_receive_checkprops(libzfs_handle_t *hdl, nvlist_t *props,
static int
zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap,
const char *originsnap, recvflags_t *flags, int infd, const char *sendfs,
nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd,
uint64_t *action_handlep, const char *finalsnap, nvlist_t *cmdprops)
nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs,
const char *finalsnap, nvlist_t *cmdprops)
{
int err;
dmu_replay_record_t drr, drr_noswap;
@ -5546,12 +5076,12 @@ zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap,
}
return (zfs_receive_one(hdl, infd, tosnap, originsnap, flags,
&drr, &drr_noswap, sendfs, stream_nv, stream_avl, top_zfs,
cleanup_fd, action_handlep, finalsnap, cmdprops));
finalsnap, cmdprops));
} else {
assert(DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
DMU_COMPOUNDSTREAM);
return (zfs_receive_package(hdl, infd, tosnap, flags, &drr,
&zcksum, top_zfs, cleanup_fd, action_handlep, cmdprops));
&zcksum, top_zfs, cmdprops));
}
}
@ -5568,8 +5098,6 @@ zfs_receive(libzfs_handle_t *hdl, const char *tosnap, nvlist_t *props,
{
char *top_zfs = NULL;
int err;
int cleanup_fd;
uint64_t action_handle = 0;
struct stat sb;
char *originsnap = NULL;
@ -5595,13 +5123,8 @@ zfs_receive(libzfs_handle_t *hdl, const char *tosnap, nvlist_t *props,
return (err);
}
cleanup_fd = open(ZFS_DEV, O_RDWR);
VERIFY(cleanup_fd >= 0);
err = zfs_receive_impl(hdl, tosnap, originsnap, flags, infd, NULL, NULL,
stream_avl, &top_zfs, cleanup_fd, &action_handle, NULL, props);
VERIFY(0 == close(cleanup_fd));
stream_avl, &top_zfs, NULL, props);
if (err == 0 && !flags->nomount && flags->domount && top_zfs) {
zfs_handle_t *zhp = NULL;

44
lib/libzfs_core/libzfs_core.c
View File

@ -20,7 +20,7 @@
*/
/*
* Copyright (c) 2012, 2018 by Delphix. All rights reserved.
* Copyright (c) 2012, 2020 by Delphix. All rights reserved.
* Copyright (c) 2013 Steven Hartland. All rights reserved.
* Copyright (c) 2017 Datto Inc.
* Copyright 2017 RackTop Systems.
@ -783,9 +783,8 @@ static int
recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops,
uint8_t *wkeydata, uint_t wkeylen, const char *origin, boolean_t force,
boolean_t resumable, boolean_t raw, int input_fd,
const dmu_replay_record_t *begin_record, int cleanup_fd,
uint64_t *read_bytes, uint64_t *errflags, uint64_t *action_handle,
nvlist_t **errors)
const dmu_replay_record_t *begin_record, uint64_t *read_bytes,
uint64_t *errflags, nvlist_t **errors)
{
dmu_replay_record_t drr;
char fsname[MAXPATHLEN];
@ -868,12 +867,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops,
if (resumable)
fnvlist_add_boolean(innvl, "resumable");
if (cleanup_fd >= 0)
fnvlist_add_int32(innvl, "cleanup_fd", cleanup_fd);
if (action_handle != NULL)
fnvlist_add_uint64(innvl, "action_handle",
*action_handle);
error = lzc_ioctl(ZFS_IOC_RECV_NEW, fsname, innvl, &outnvl);
@ -885,10 +878,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops,
error = nvlist_lookup_uint64(outnvl, "error_flags",
errflags);
if (error == 0 && action_handle != NULL)
error = nvlist_lookup_uint64(outnvl, "action_handle",
action_handle);
if (error == 0 && errors != NULL) {
nvlist_t *nvl;
error = nvlist_lookup_nvlist(outnvl, "errors", &nvl);
@ -931,12 +920,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops,
zc.zc_cleanup_fd = -1;
zc.zc_action_handle = 0;
if (cleanup_fd >= 0)
zc.zc_cleanup_fd = cleanup_fd;
if (action_handle != NULL)
zc.zc_action_handle = *action_handle;
zc.zc_nvlist_dst_size = 128 * 1024;
zc.zc_nvlist_dst = (uint64_t)(uintptr_t)
malloc(zc.zc_nvlist_dst_size);
@ -951,9 +934,6 @@ recv_impl(const char *snapname, nvlist_t *recvdprops, nvlist_t *localprops,
if (errflags != NULL)
*errflags = zc.zc_obj;
if (action_handle != NULL)
*action_handle = zc.zc_action_handle;
if (errors != NULL)
VERIFY0(nvlist_unpack(
(void *)(uintptr_t)zc.zc_nvlist_dst,
@ -986,7 +966,7 @@ lzc_receive(const char *snapname, nvlist_t *props, const char *origin,
boolean_t force, boolean_t raw, int fd)
{
return (recv_impl(snapname, props, NULL, NULL, 0, origin, force,
B_FALSE, raw, fd, NULL, -1, NULL, NULL, NULL, NULL));
B_FALSE, raw, fd, NULL, NULL, NULL, NULL));
}
/*
@ -1000,7 +980,7 @@ lzc_receive_resumable(const char *snapname, nvlist_t *props, const char *origin,
boolean_t force, boolean_t raw, int fd)
{
return (recv_impl(snapname, props, NULL, NULL, 0, origin, force,
B_TRUE, raw, fd, NULL, -1, NULL, NULL, NULL, NULL));
B_TRUE, raw, fd, NULL, NULL, NULL, NULL));
}
/*
@ -1023,7 +1003,7 @@ lzc_receive_with_header(const char *snapname, nvlist_t *props,
return (EINVAL);
return (recv_impl(snapname, props, NULL, NULL, 0, origin, force,
resumable, raw, fd, begin_record, -1, NULL, NULL, NULL, NULL));
resumable, raw, fd, begin_record, NULL, NULL, NULL));
}
/*
@ -1039,9 +1019,7 @@ lzc_receive_with_header(const char *snapname, nvlist_t *props,
* The 'errflags' value will contain zprop_errflags_t flags which are
* used to describe any failures.
*
* The 'action_handle' is used to pass the handle for this guid/ds mapping.
* It should be set to zero on first call and will contain an updated handle
* on success, it should be passed in subsequent calls.
* The 'action_handle' and 'cleanup_fd' are no longer used, and are ignored.
*
* The 'errors' nvlist contains an entry for each unapplied received
* property. Callers are responsible for freeing this nvlist.
@ -1053,8 +1031,8 @@ int lzc_receive_one(const char *snapname, nvlist_t *props,
nvlist_t **errors)
{
return (recv_impl(snapname, props, NULL, NULL, 0, origin, force,
resumable, raw, input_fd, begin_record, cleanup_fd, read_bytes,
errflags, action_handle, errors));
resumable, raw, input_fd, begin_record,
read_bytes, errflags, errors));
}
/*
@ -1073,8 +1051,8 @@ int lzc_receive_with_cmdprops(const char *snapname, nvlist_t *props,
nvlist_t **errors)
{
return (recv_impl(snapname, props, cmdprops, wkeydata, wkeylen, origin,
force, resumable, raw, input_fd, begin_record, cleanup_fd,
read_bytes, errflags, action_handle, errors));
force, resumable, raw, input_fd, begin_record,
read_bytes, errflags, errors));
}
/*

15
man/man8/zfs-receive.8
View File

@ -105,17 +105,11 @@ destroyed by using the
.Nm zfs Cm destroy Fl d
command.
.Pp
Deduplicated send streams can be generated by using the
.Nm zfs Cm send Fl D
The ability to send and receive deduplicated send streams has been removed.
However, a deduplicated send stream created with older software can be converted
to a regular (non-deduplicated) stream by using the
.Nm zstream Cm redup
command.
\fBThe ability to send and receive deduplicated send streams is deprecated.\fR
In the future, the ability to receive a deduplicated send stream with
.Nm zfs Cm receive
will be removed.
However, in the future, a utility will be provided to convert a
deduplicated send stream to a regular (non-deduplicated) stream.
This future utility will require that the send stream be located in a
seek-able file, rather than provided by a pipe.
.Pp
If
.Fl o Em property Ns = Ns Ar value
@ -378,3 +372,4 @@ deleting its saved partially received state.
.El
.Sh SEE ALSO
.Xr zfs-send 8
.Xr zstream 8

18
man/man8/zfs-send.8
View File

@ -86,21 +86,9 @@ The output can be redirected to a file or to a different system
By default, a full stream is generated.
.Bl -tag -width "-D"
.It Fl D, -dedup
Generate a deduplicated stream.
\fBDeduplicated send is deprecated and will be removed in a future release.\fR
(In the future, the flag will be accepted but a regular, non-deduplicated
stream will be generated.)
Blocks which would have been sent multiple times in the send stream will only be
sent once.
The receiving system must also support this feature to receive a deduplicated
stream.
This flag can be used regardless of the dataset's
.Sy dedup
property, but performance will be much better if the filesystem uses a
dedup-capable checksum
.Po for example,
.Sy sha256
.Pc .
Deduplicated send is no longer supported.
This flag is accepted for backwards compatibility, but a regular,
non-deduplicated stream will be generated.
.It Fl I Ar snapshot
Generate a stream package that sends all intermediary snapshots from the first
snapshot to the second snapshot.

52
module/zfs/dmu.c
View File

@ -20,7 +20,7 @@
*/
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011, 2018 by Delphix. All rights reserved.
* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
* Copyright (c) 2013, Joyent, Inc. All rights reserved.
* Copyright (c) 2016, Nexenta Systems, Inc. All rights reserved.
@ -1560,56 +1560,6 @@ dmu_return_arcbuf(arc_buf_t *buf)
arc_buf_destroy(buf, FTAG);
}
void
dmu_copy_from_buf(objset_t *os, uint64_t object, uint64_t offset,
dmu_buf_t *handle, dmu_tx_t *tx)
{
dmu_buf_t *dst_handle;
dmu_buf_impl_t *dstdb;
dmu_buf_impl_t *srcdb = (dmu_buf_impl_t *)handle;
dmu_object_type_t type;
arc_buf_t *abuf;
uint64_t datalen;
boolean_t byteorder;
uint8_t salt[ZIO_DATA_SALT_LEN];
uint8_t iv[ZIO_DATA_IV_LEN];
uint8_t mac[ZIO_DATA_MAC_LEN];
ASSERT3P(srcdb->db_buf, !=, NULL);
/* hold the db that we want to write to */
VERIFY0(dmu_buf_hold(os, object, offset, FTAG, &dst_handle,
DMU_READ_NO_DECRYPT));
dstdb = (dmu_buf_impl_t *)dst_handle;
datalen = arc_buf_size(srcdb->db_buf);
DB_DNODE_ENTER(dstdb);
type = DB_DNODE(dstdb)->dn_type;
DB_DNODE_EXIT(dstdb);
/* allocated an arc buffer that matches the type of srcdb->db_buf */
if (arc_is_encrypted(srcdb->db_buf)) {
arc_get_raw_params(srcdb->db_buf, &byteorder, salt, iv, mac);
abuf = arc_loan_raw_buf(os->os_spa, dmu_objset_id(os),
byteorder, salt, iv, mac, type,
datalen, arc_buf_lsize(srcdb->db_buf),
arc_get_compression(srcdb->db_buf));
} else {
/* we won't get a compressed db back from dmu_buf_hold() */
ASSERT3U(arc_get_compression(srcdb->db_buf),
==, ZIO_COMPRESS_OFF);
abuf = arc_loan_buf(os->os_spa,
DMU_OT_IS_METADATA(type), datalen);
}
ASSERT3U(datalen, ==, arc_buf_size(abuf));
/* copy the data to the new buffer and assign it to the dstdb */
bcopy(srcdb->db_buf->b_data, abuf->b_data, datalen);
dbuf_assign_arcbuf(dstdb, abuf, tx);
dmu_buf_rele(dst_handle, FTAG);
}
/*
* When possible directly assign passed loaned arc buffer to a dbuf.
* If this is not possible copy the contents of passed arc buf via

229
module/zfs/dmu_recv.c
View File

@ -21,7 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright 2011 Nexenta Systems, Inc. All rights reserved.
* Copyright (c) 2011, 2018 by Delphix. All rights reserved.
* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
* Copyright (c) 2014, Joyent, Inc. All rights reserved.
* Copyright 2014 HybridCluster. All rights reserved.
* Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
@ -101,8 +101,6 @@ struct receive_writer_arg {
boolean_t done;
int err;
/* A map from guid to dataset to help handle dedup'd streams. */
avl_tree_t *guid_to_ds_map;
boolean_t resumable;
boolean_t raw; /* DMU_BACKUP_FEATURE_RAW set */
boolean_t spill; /* DRR_FLAG_SPILL_BLOCK set */
@ -123,13 +121,6 @@ struct receive_writer_arg {
boolean_t or_byteorder;
};
typedef struct guid_map_entry {
uint64_t guid;
boolean_t raw;
objset_t *gme_os;
avl_node_t avlnode;
} guid_map_entry_t;
typedef struct dmu_recv_begin_arg {
const char *drba_origin;
dmu_recv_cookie_t *drba_cookie;
@ -1212,38 +1203,6 @@ dmu_recv_begin(char *tofs, char *tosnap, dmu_replay_record_t *drr_begin,
return (err);
}
static int
guid_compare(const void *arg1, const void *arg2)
{
const guid_map_entry_t *gmep1 = (const guid_map_entry_t *)arg1;
const guid_map_entry_t *gmep2 = (const guid_map_entry_t *)arg2;
return (TREE_CMP(gmep1->guid, gmep2->guid));
}
static void
free_guid_map_onexit(void *arg)
{
avl_tree_t *ca = arg;
void *cookie = NULL;
guid_map_entry_t *gmep;
while ((gmep = avl_destroy_nodes(ca, &cookie)) != NULL) {
ds_hold_flags_t dsflags = DS_HOLD_FLAG_DECRYPT;
if (gmep->raw) {
gmep->gme_os->os_raw_receive = B_FALSE;
dsflags &= ~DS_HOLD_FLAG_DECRYPT;
}
dsl_dataset_disown(gmep->gme_os->os_dsl_dataset,
dsflags, gmep);
kmem_free(gmep, sizeof (guid_map_entry_t));
}
avl_destroy(ca);
kmem_free(ca, sizeof (avl_tree_t));
}
static int
receive_read(dmu_recv_cookie_t *drc, int len, void *buf)
{
@ -1845,81 +1804,6 @@ receive_process_write_record(struct receive_writer_arg *rwa,
return (EAGAIN);
}
/*
* Handle a DRR_WRITE_BYREF record. This record is used in dedup'ed
* streams to refer to a copy of the data that is already on the
* system because it came in earlier in the stream. This function
* finds the earlier copy of the data, and uses that copy instead of
* data from the stream to fulfill this write.
*/
noinline static int
receive_write_byref(struct receive_writer_arg *rwa,
struct drr_write_byref *drrwbr)
{
dmu_tx_t *tx;
int err;
guid_map_entry_t gmesrch;
guid_map_entry_t *gmep;
avl_index_t where;
objset_t *ref_os = NULL;
int flags = DMU_READ_PREFETCH;
dmu_buf_t *dbp;
if (drrwbr->drr_offset + drrwbr->drr_length < drrwbr->drr_offset)
return (SET_ERROR(EINVAL));
/*
* If the GUID of the referenced dataset is different from the
* GUID of the target dataset, find the referenced dataset.
*/
if (drrwbr->drr_toguid != drrwbr->drr_refguid) {
gmesrch.guid = drrwbr->drr_refguid;
if ((gmep = avl_find(rwa->guid_to_ds_map, &gmesrch,
&where)) == NULL) {
return (SET_ERROR(EINVAL));
}
ref_os = gmep->gme_os;
} else {
ref_os = rwa->os;
}
if (drrwbr->drr_object > rwa->max_object)
rwa->max_object = drrwbr->drr_object;
if (rwa->raw)
flags |= DMU_READ_NO_DECRYPT;
/* may return either a regular db or an encrypted one */
err = dmu_buf_hold(ref_os, drrwbr->drr_refobject,
drrwbr->drr_refoffset, FTAG, &dbp, flags);
if (err != 0)
return (err);
tx = dmu_tx_create(rwa->os);
dmu_tx_hold_write(tx, drrwbr->drr_object,
drrwbr->drr_offset, drrwbr->drr_length);
err = dmu_tx_assign(tx, TXG_WAIT);
if (err != 0) {
dmu_tx_abort(tx);
return (err);
}
if (rwa->raw) {
dmu_copy_from_buf(rwa->os, drrwbr->drr_object,
drrwbr->drr_offset, dbp, tx);
} else {
dmu_write(rwa->os, drrwbr->drr_object,
drrwbr->drr_offset, drrwbr->drr_length, dbp->db_data, tx);
}
dmu_buf_rele(dbp, FTAG);
/* See comment in restore_write. */
save_resume_state(rwa, drrwbr->drr_object, drrwbr->drr_offset, tx);
dmu_tx_commit(tx);
return (0);
}
static int
receive_write_embedded(struct receive_writer_arg *rwa,
struct drr_write_embedded *drrwe, void *data)
@ -2607,13 +2491,6 @@ receive_process_record(struct receive_writer_arg *rwa,
}
break;
}
case DRR_WRITE_BYREF:
{
struct drr_write_byref *drrwbr =
&rrd->header.drr_u.drr_write_byref;
err = receive_write_byref(rwa, drrwbr);
break;
}
case DRR_WRITE_EMBEDDED:
{
struct drr_write_embedded *drrwe =
@ -2754,8 +2631,7 @@ resume_check(dmu_recv_cookie_t *drc, nvlist_t *begin_nvl)
* NB: callers *must* call dmu_recv_end() if this succeeds.
*/
int
dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd,
uint64_t *action_handlep, offset_t *voffp)
dmu_recv_stream(dmu_recv_cookie_t *drc, offset_t *voffp)
{
int err = 0;
struct receive_writer_arg *rwa = kmem_zalloc(sizeof (*rwa), KM_SLEEP);
@ -2779,41 +2655,6 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd,
ASSERT0(drc->drc_os->os_encrypted &&
(drc->drc_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA));
/* if this stream is dedup'ed, set up the avl tree for guid mapping */
if (drc->drc_featureflags & DMU_BACKUP_FEATURE_DEDUP) {
minor_t minor;
if (cleanup_fd == -1) {
err = SET_ERROR(EBADF);
goto out;
}
err = zfs_onexit_fd_hold(cleanup_fd, &minor);
if (err != 0) {
cleanup_fd = -1;
goto out;
}
if (*action_handlep == 0) {
rwa->guid_to_ds_map =
kmem_alloc(sizeof (avl_tree_t), KM_SLEEP);
avl_create(rwa->guid_to_ds_map, guid_compare,
sizeof (guid_map_entry_t),
offsetof(guid_map_entry_t, avlnode));
err = zfs_onexit_add_cb(minor,
free_guid_map_onexit, rwa->guid_to_ds_map,
action_handlep);
if (err != 0)
goto out;
} else {
err = zfs_onexit_cb_data(minor, *action_handlep,
(void **)&rwa->guid_to_ds_map);
if (err != 0)
goto out;
}
drc->drc_guid_to_ds_map = rwa->guid_to_ds_map;
}
/* handle DSL encryption key payload */
if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) {
nvlist_t *keynvl = NULL;
@ -2980,9 +2821,6 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, int cleanup_fd,
kmem_free(rwa, sizeof (*rwa));
nvlist_free(drc->drc_begin_nvl);
if ((drc->drc_featureflags & DMU_BACKUP_FEATURE_DEDUP) &&
(cleanup_fd != -1))
zfs_onexit_fd_rele(cleanup_fd);
if (err != 0) {
/*
@ -3083,6 +2921,7 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
dmu_recv_cookie_t *drc = arg;
dsl_pool_t *dp = dmu_tx_pool(tx);
boolean_t encrypted = drc->drc_ds->ds_dir->dd_crypto_obj != 0;
uint64_t newsnapobj;
spa_history_log_internal_ds(drc->drc_ds, "finish receiving",
tx, "snap=%s", drc->drc_tosnap);
@ -3148,7 +2987,7 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
dsl_dataset_phys(origin_head)->ds_flags &=
~DS_FLAG_INCONSISTENT;
drc->drc_newsnapobj =
newsnapobj =
dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
dsl_dataset_rele(origin_head, FTAG);
@ -3188,7 +3027,7 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, tx);
}
drc->drc_newsnapobj =
newsnapobj =
dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj;
}
@ -3203,9 +3042,9 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
* value.
*/
if (drc->drc_raw && drc->drc_ivset_guid != 0) {
dmu_object_zapify(dp->dp_meta_objset, drc->drc_newsnapobj,
dmu_object_zapify(dp->dp_meta_objset, newsnapobj,
DMU_OT_DSL_DATASET, tx);
VERIFY0(zap_update(dp->dp_meta_objset, drc->drc_newsnapobj,
VERIFY0(zap_update(dp->dp_meta_objset, newsnapobj,
DS_FIELD_IVSET_GUID, sizeof (uint64_t), 1,
&drc->drc_ivset_guid, tx));
}
@ -3226,54 +3065,6 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
drc->drc_ds = NULL;
}
static int
add_ds_to_guidmap(const char *name, avl_tree_t *guid_map, uint64_t snapobj,
boolean_t raw)
{
dsl_pool_t *dp;
dsl_dataset_t *snapds;
guid_map_entry_t *gmep;
objset_t *os;
ds_hold_flags_t dsflags = (raw) ? 0 : DS_HOLD_FLAG_DECRYPT;
int err;
ASSERT(guid_map != NULL);
err = dsl_pool_hold(name, FTAG, &dp);
if (err != 0)
return (err);
gmep = kmem_alloc(sizeof (*gmep), KM_SLEEP);
err = dsl_dataset_own_obj(dp, snapobj, dsflags, gmep, &snapds);
if (err == 0) {
err = dmu_objset_from_ds(snapds, &os);
if (err != 0) {
dsl_dataset_disown(snapds, dsflags, FTAG);
dsl_pool_rele(dp, FTAG);
kmem_free(gmep, sizeof (*gmep));
return (err);
}
/*
* If this is a deduplicated raw send stream, we need
* to make sure that we can still read raw blocks from
* earlier datasets in the stream, so we set the
* os_raw_receive flag now.
*/
if (raw)
os->os_raw_receive = B_TRUE;
gmep->raw = raw;
gmep->guid = dsl_dataset_phys(snapds)->ds_guid;
gmep->gme_os = os;
avl_add(guid_map, gmep);
} else {
kmem_free(gmep, sizeof (*gmep));
}
dsl_pool_rele(dp, FTAG);
return (err);
}
static int dmu_recv_end_modified_blocks = 3;
static int
@ -3325,12 +3116,6 @@ dmu_recv_end(dmu_recv_cookie_t *drc, void *owner)
drc->drc_tofs, drc->drc_tosnap);
zvol_create_minor(snapname);
kmem_strfree(snapname);
if (drc->drc_guid_to_ds_map != NULL) {
(void) add_ds_to_guidmap(drc->drc_tofs,
drc->drc_guid_to_ds_map,
drc->drc_newsnapobj, drc->drc_raw);
}
}
return (error);
}

36
module/zfs/zfs_ioctl.c
View File

@ -27,7 +27,7 @@
* Copyright (c) 2014, 2016 Joyent, Inc. All rights reserved.
* Copyright 2016 Nexenta Systems, Inc. All rights reserved.
* Copyright (c) 2014, Joyent, Inc. All rights reserved.
* Copyright (c) 2011, 2018 by Delphix. All rights reserved.
* Copyright (c) 2011, 2020 by Delphix. All rights reserved.
* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
* Copyright (c) 2013 Steven Hartland. All rights reserved.
* Copyright (c) 2014 Integros [integros.com]
@ -4765,9 +4765,9 @@ static boolean_t zfs_ioc_recv_inject_err;
static int
zfs_ioc_recv_impl(char *tofs, char *tosnap, char *origin, nvlist_t *recvprops,
nvlist_t *localprops, nvlist_t *hidden_args, boolean_t force,
boolean_t resumable, int input_fd, dmu_replay_record_t *begin_record,
int cleanup_fd, uint64_t *read_bytes, uint64_t *errflags,
uint64_t *action_handle, nvlist_t **errors)
boolean_t resumable, int input_fd,
dmu_replay_record_t *begin_record, uint64_t *read_bytes,
uint64_t *errflags, nvlist_t **errors)
{
dmu_recv_cookie_t drc;
int error = 0;
@ -4896,7 +4896,7 @@ zfs_ioc_recv_impl(char *tofs, char *tosnap, char *origin, nvlist_t *recvprops,
nvlist_free(xprops);
}
error = dmu_recv_stream(&drc, cleanup_fd, action_handle, &off);
error = dmu_recv_stream(&drc, &off);
if (error == 0) {
zfsvfs_t *zfsvfs = NULL;
@ -5088,13 +5088,10 @@ zfs_ioc_recv_impl(char *tofs, char *tosnap, char *origin, nvlist_t *recvprops,
* zc_cookie file descriptor to recv from
* zc_begin_record the BEGIN record of the stream (not byteswapped)
* zc_guid force flag
* zc_cleanup_fd cleanup-on-exit file descriptor
* zc_action_handle handle for this guid/ds mapping (or zero on first call)
*
* outputs:
* zc_cookie number of bytes read
* zc_obj zprop_errflags_t
* zc_action_handle handle for this guid/ds mapping
* zc_nvlist_dst{_size} error for each unapplied received property
*/
static int
@ -5137,8 +5134,7 @@ zfs_ioc_recv(zfs_cmd_t *zc)
error = zfs_ioc_recv_impl(tofs, tosnap, origin, recvdprops, localprops,
NULL, zc->zc_guid, B_FALSE, zc->zc_cookie, &begin_record,
zc->zc_cleanup_fd, &zc->zc_cookie, &zc->zc_obj,
&zc->zc_action_handle, &errors);
&zc->zc_cookie, &zc->zc_obj, &errors);
nvlist_free(recvdprops);
nvlist_free(localprops);
@ -5171,15 +5167,14 @@ zfs_ioc_recv(zfs_cmd_t *zc)
* "input_fd" -> file descriptor to read stream from (int32)
* (optional) "force" -> force flag (value ignored)
* (optional) "resumable" -> resumable flag (value ignored)
* (optional) "cleanup_fd" -> cleanup-on-exit file descriptor
* (optional) "action_handle" -> handle for this guid/ds mapping
* (optional) "cleanup_fd" -> unused
* (optional) "action_handle" -> unused
* (optional) "hidden_args" -> { "wkeydata" -> value }
* }
*
* outnvl: {
* "read_bytes" -> number of bytes read
* "error_flags" -> zprop_errflags_t
* "action_handle" -> handle for this guid/ds mapping
* "errors" -> error for each unapplied received property (nvlist)
* }
*/
@ -5212,11 +5207,9 @@ zfs_ioc_recv_new(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl)
char tofs[ZFS_MAX_DATASET_NAME_LEN];
boolean_t force;
boolean_t resumable;
uint64_t action_handle = 0;
uint64_t read_bytes = 0;
uint64_t errflags = 0;
int input_fd = -1;
int cleanup_fd = -1;
int error;
snapname = fnvlist_lookup_string(innvl, "snapname");
@ -5244,14 +5237,6 @@ zfs_ioc_recv_new(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl)
force = nvlist_exists(innvl, "force");
resumable = nvlist_exists(innvl, "resumable");
error = nvlist_lookup_int32(innvl, "cleanup_fd", &cleanup_fd);
if (error && error != ENOENT)
return (error);
error = nvlist_lookup_uint64(innvl, "action_handle", &action_handle);
if (error && error != ENOENT)
return (error);
/* we still use "props" here for backwards compatibility */
error = nvlist_lookup_nvlist(innvl, "props", &recvprops);
if (error && error != ENOENT)
@ -5266,12 +5251,11 @@ zfs_ioc_recv_new(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl)
return (error);
error = zfs_ioc_recv_impl(tofs, tosnap, origin, recvprops, localprops,
hidden_args, force, resumable, input_fd, begin_record, cleanup_fd,
&read_bytes, &errflags, &action_handle, &errors);
hidden_args, force, resumable, input_fd, begin_record,
&read_bytes, &errflags, &errors);
fnvlist_add_uint64(outnvl, "read_bytes", read_bytes);
fnvlist_add_uint64(outnvl, "error_flags", errflags);
fnvlist_add_uint64(outnvl, "action_handle", action_handle);
fnvlist_add_nvlist(outnvl, "errors", errors);
nvlist_free(errors);

79
module/zfs/zfs_onexit.c
View File

@ -20,7 +20,7 @@
*/
/*
* Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013 by Delphix. All rights reserved.
* Copyright (c) 2013, 2020 by Delphix. All rights reserved.
*/
#include <sys/types.h>
@ -171,80 +171,3 @@ zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
return (0);
}
static zfs_onexit_action_node_t *
zfs_onexit_find_cb(zfs_onexit_t *zo, uint64_t action_handle)
{
zfs_onexit_action_node_t *match;
zfs_onexit_action_node_t *ap;
list_t *l;
ASSERT(MUTEX_HELD(&zo->zo_lock));
match = (zfs_onexit_action_node_t *)(uintptr_t)action_handle;
l = &zo->zo_actions;
for (ap = list_head(l); ap != NULL; ap = list_next(l, ap)) {
if (match == ap)
break;
}
return (ap);
}
/*
* Delete the callback, triggering it first if 'fire' is set.
*/
int
zfs_onexit_del_cb(minor_t minor, uint64_t action_handle, boolean_t fire)
{
zfs_onexit_t *zo;
zfs_onexit_action_node_t *ap;
int error;
error = zfs_onexit_minor_to_state(minor, &zo);
if (error)
return (error);
mutex_enter(&zo->zo_lock);
ap = zfs_onexit_find_cb(zo, action_handle);
if (ap != NULL) {
list_remove(&zo->zo_actions, ap);
mutex_exit(&zo->zo_lock);
if (fire)
ap->za_func(ap->za_data);
kmem_free(ap, sizeof (zfs_onexit_action_node_t));
} else {
mutex_exit(&zo->zo_lock);
error = SET_ERROR(ENOENT);
}
return (error);
}
/*
* Return the data associated with this callback. This allows consumers
* of the cleanup-on-exit interfaces to stash kernel data across system
* calls, knowing that it will be cleaned up if the calling process exits.
*/
int
zfs_onexit_cb_data(minor_t minor, uint64_t action_handle, void **data)
{
zfs_onexit_t *zo;
zfs_onexit_action_node_t *ap;
int error;
*data = NULL;
error = zfs_onexit_minor_to_state(minor, &zo);
if (error)
return (error);
mutex_enter(&zo->zo_lock);
ap = zfs_onexit_find_cb(zo, action_handle);
if (ap != NULL)
*data = ap->za_data;
else
error = SET_ERROR(ENOENT);
mutex_exit(&zo->zo_lock);
return (error);
}

14
tests/runfiles/common.run
View File

@ -766,22 +766,22 @@ tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_pos']
tags = ['functional', 'rootpool']
[tests/functional/rsend]
tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos',
'rsend_005_pos', 'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos',
'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos',
'rsend_013_pos', 'rsend_014_pos', 'rsend_016_neg',
'rsend_019_pos', 'rsend_020_pos',
tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos',
'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', 'rsend_005_pos',
'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos', 'rsend_009_pos',
'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', 'rsend_013_pos',
'rsend_014_pos', 'rsend_016_neg', 'rsend_019_pos', 'rsend_020_pos',
'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos',
'send-c_verify_ratio', 'send-c_verify_contents', 'send-c_props',
'send-c_incremental', 'send-c_volume', 'send-c_zstreamdump',
'send-c_lz4_disabled', 'send-c_recv_lz4_disabled',
'send-c_mixed_compression', 'send-c_stream_size_estimate', 'send-cD',
'send-c_mixed_compression', 'send-c_stream_size_estimate',
'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
'send-c_recv_dedup', 'send_encrypted_hierarchy',
'send_encrypted_props', 'send_encrypted_truncated_files',
'send_freeobjects', 'send_realloc_files',
'send_realloc_encrypted_files', 'send_spill_block', 'send_holds',
'send_hole_birth', 'send_mixed_raw', 'send-wDR_encrypted_zvol',
'send_hole_birth', 'send_mixed_raw', 'send-wR_encrypted_zvol',
'send_partial_dataset']
tags = ['functional', 'rsend']

9
tests/zfs-tests/tests/functional/rsend/Makefile.am
View File

@ -2,6 +2,8 @@ pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/rsend
dist_pkgdata_SCRIPTS = \
setup.ksh \
cleanup.ksh \
recv_dedup.ksh \
recv_dedup_encrypted_zvol.ksh \
rsend_001_pos.ksh \
rsend_002_pos.ksh \
rsend_003_pos.ksh \
@ -25,7 +27,6 @@ dist_pkgdata_SCRIPTS = \
send_encrypted_hierarchy.ksh \
send_encrypted_props.ksh \
send_encrypted_truncated_files.ksh \
send-cD.ksh \
send-c_embedded_blocks.ksh \
send-c_incremental.ksh \
send-c_lz4_disabled.ksh \
@ -49,8 +50,12 @@ dist_pkgdata_SCRIPTS = \
send_holds.ksh \
send_hole_birth.ksh \
send_mixed_raw.ksh \
send-wDR_encrypted_zvol.ksh
send-wR_encrypted_zvol.ksh
dist_pkgdata_DATA = \
dedup.zsend.bz2 \
dedup_encrypted_zvol.bz2 \
dedup_encrypted_zvol.zsend.bz2 \
fs.tar.gz \
rsend.cfg \
rsend.kshlib

BIN
tests/zfs-tests/tests/functional/rsend/dedup.zsend.bz2 Normal file
View File

Binary file not shown.

BIN
tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.bz2 Normal file
View File

Binary file not shown.

BIN
tests/zfs-tests/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2 Normal file
View File

Binary file not shown.

BIN
tests/zfs-tests/tests/functional/rsend/fs.tar.gz Normal file
View File

Binary file not shown.

53
tests/zfs-tests/tests/functional/rsend/recv_dedup.ksh Executable file
View File

@ -0,0 +1,53 @@
#!/bin/ksh -p
#
# CDDL HEADER START
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
# CDDL HEADER END
#
#
# Copyright (c) 2020 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
#
# DESCRIPTION:
# Verifies that we can receive a dedup send stream by processing it with
# "zstream redup".
#
verify_runnable "both"
function cleanup
{
destroy_dataset $TESTPOOL/recv "-r"
rm -r /$TESTPOOL/tar
rm $sendfile
}
log_onexit cleanup
log_assert "Verify zfs can receive dedup send streams with 'zstream redup'"
typeset sendfile_compressed=$STF_SUITE/tests/functional/rsend/dedup.zsend.bz2
typeset sendfile=/$TESTPOOL/dedup.zsend
typeset tarfile=$STF_SUITE/tests/functional/rsend/fs.tar.gz
log_must eval "bzcat <$sendfile_compressed >$sendfile"
log_must zfs create $TESTPOOL/recv
log_must eval "zstream redup $sendfile | zfs recv -d $TESTPOOL/recv"
log_must mkdir /$TESTPOOL/tar
log_must tar --directory /$TESTPOOL/tar -xzf $tarfile
log_must diff -r /$TESTPOOL/tar /$TESTPOOL/recv
log_pass "zfs can receive dedup send streams with 'zstream redup'"

60
tests/zfs-tests/tests/functional/rsend/recv_dedup_encrypted_zvol.ksh Executable file
View File

@ -0,0 +1,60 @@
#!/bin/ksh -p
#
# CDDL HEADER START
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
# CDDL HEADER END
#
#
# Copyright (c) 2020 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
#
# DESCRIPTION:
# Verifies that we can receive a dedup send stream of a zvol by processing it
# with "zstream redup".
#
verify_runnable "both"
function cleanup
{
destroy_dataset $TESTPOOL/recv "-r"
rm $sendfile
rm $volfile
rm $keyfile
}
log_onexit cleanup
log_assert "Verify zfs can receive raw, recursive, and deduplicated send streams"
typeset keyfile=/$TESTPOOL/pkey
typeset recvdev=$ZVOL_DEVDIR/$TESTPOOL/recv
typeset sendfile_compressed=$STF_SUITE/tests/functional/rsend/dedup_encrypted_zvol.zsend.bz2
typeset sendfile=/$TESTPOOL/dedup_encrypted_zvol.zsend
typeset volfile_compressed=$STF_SUITE/tests/functional/rsend/dedup_encrypted_zvol.bz2
typeset volfile=/$TESTPOOL/dedup_encrypted_zvol
log_must eval "echo 'password' > $keyfile"
log_must eval "bzcat <$sendfile_compressed >$sendfile"
log_must eval "zstream redup $sendfile | zfs recv $TESTPOOL/recv"
log_must zfs load-key $TESTPOOL/recv
block_device_wait
log_must eval "bzcat <$volfile_compressed >$volfile"
log_must diff $volfile $recvdev
log_pass "zfs can receive raw, recursive, and deduplicated send streams"

89
tests/zfs-tests/tests/functional/rsend/send-cD.ksh
View File

@ -1,89 +0,0 @@
#!/bin/ksh -p
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source. A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#
#
# Copyright (c) 2015, 2018 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
#
# Description:
# Verify that the -c and -D flags do not interfere with each other.
#
# Strategy:
# 1. Write unique data to a filesystem and create a compressed, deduplicated
# full stream.
# 2. Verify that the stream and send dataset show the same size
# 3. Make several copies of the original data, and create both full and
# incremental compressed, deduplicated send streams
# 4. Verify the full stream is no bigger than the stream from step 1
# 5. Verify the streams can be received correctly.
#
verify_runnable "both"
log_assert "Verify that the -c and -D flags do not interfere with each other"
log_onexit cleanup_pool $POOL2
typeset sendfs=$POOL2/sendfs
typeset recvfs=$POOL2/recvfs
typeset stream0=$BACKDIR/stream.0
typeset stream1=$BACKDIR/stream.1
typeset inc=$BACKDIR/stream.inc
log_must zfs create -o compress=lz4 $sendfs
log_must zfs create -o compress=lz4 $recvfs
typeset dir=$(get_prop mountpoint $sendfs)
# Don't use write_compressible: we want compressible but undeduplicable data.
log_must eval "dd if=/dev/urandom bs=1024k count=4 | base64 >$dir/file"
log_must zfs snapshot $sendfs@snap0
log_must eval "zfs send -D -c $sendfs@snap0 >$stream0"
# The stream size should match at this point because the data is all unique
verify_stream_size $stream0 $sendfs
for i in {0..3}; do
log_must cp $dir/file $dir/file.$i
done
log_must zfs snapshot $sendfs@snap1
# The stream sizes should match, since the second stream contains no new blocks
log_must eval "zfs send -D -c $sendfs@snap1 >$stream1"
typeset size0=$(stat_size $stream0)
typeset size1=$(stat_size $stream1)
within_percent $size0 $size1 90 || log_fail "$size0 and $size1"
# make sure the receive works correctly.
log_must eval "zfs send -D -c -i snap0 $sendfs@snap1 >$inc"
log_must eval "zfs recv -d $recvfs <$stream0"
log_must eval "zfs recv -d $recvfs <$inc"
cmp_ds_cont $sendfs $recvfs
# check receive with redup.
log_must zfs destroy -r $recvfs
log_must zfs create -o compress=lz4 $recvfs
log_must eval "zstream redup $stream0 | zfs recv -d $recvfs"
log_must eval "zstream redup $inc | zfs recv -d $recvfs"
cmp_ds_cont $sendfs $recvfs
# The size of the incremental should be the same as the initial send.
typeset size2=$(stat_size $inc)
within_percent $size0 $size2 90 || log_fail "$size0 and $size1"
# The redup'ed size should be 4x
typeset size3=$(zstream redup $inc | wc -c)
let size4=size0*4
within_percent $size4 $size3 90 || log_fail "$size4 and $size3"
log_pass "The -c and -D flags do not interfere with each other"

15
tests/zfs-tests/tests/functional/rsend/send-wDR_encrypted_zvol.ksh → tests/zfs-tests/tests/functional/rsend/send-wR_encrypted_zvol.ksh
View File

@ -16,20 +16,21 @@
#
# Copyright (c) 2018 by Datto Inc. All rights reserved.
# Copyright (c) 2020 by Delphix. All rights reserved.
#
. $STF_SUITE/tests/functional/rsend/rsend.kshlib
#
# DESCRIPTION:
# Verify that zvols with dedup=on and encryption=on can be sent and received
# with a deduplicated raw send stream.
# Verify that zvols with encryption=on can be sent and received with a raw
# send stream.
#
# STRATEGY:
# 1. Create a zvol with dedup and encryption on and put a filesystem on it
# 1. Create a zvol with encryption on and put a filesystem on it
# 2. Copy a file into the zvol a few times and take a snapshot
# 3. Repeat step 2 a few times to create more snapshots
# 4. Send all snapshots in a recursive, raw, deduplicated send stream
# 4. Send all snapshots in a recursive, raw send stream
# 5. Mount the received zvol and verify that all of the data there is correct
#
@ -48,7 +49,7 @@ function cleanup
}
log_onexit cleanup
log_assert "Verify zfs can receive raw, recursive, and deduplicated send streams"
log_assert "Verify zfs can receive raw, recursive send streams"
typeset keyfile=/$TESTPOOL/pkey
typeset snap_count=5
@ -93,7 +94,7 @@ for ((i = 1; i <= $snap_count; i++)); do
log_must mount $remount_rw $zdev $mntpnt
done
log_must eval "zfs send -wDR $TESTPOOL/$TESTVOL@snap$snap_count > $sendfile"
log_must eval "zfs send -wR $TESTPOOL/$TESTVOL@snap$snap_count > $sendfile"
log_must eval "zfs recv $TESTPOOL/recv < $sendfile"
log_must zfs load-key $TESTPOOL/recv
block_device_wait
@ -104,4 +105,4 @@ md5_1=$(cat $mntpnt/* | md5digest)
md5_2=$(cat $recvmnt/* | md5digest)
[[ "$md5_1" == "$md5_2" ]] || log_fail "md5 mismatch: $md5_1 != $md5_2"
log_pass "zfs can receive raw, recursive, and deduplicated send streams"
log_pass "zfs can receive raw, recursive send streams"

2
tests/zfs-tests/tests/functional/rsend/send_partial_dataset.ksh
View File

@ -13,6 +13,7 @@
#
# Copyright (c) 2019 Datto Inc.
# Copyright (c) 2020 by Delphix. All rights reserved.
#
. $STF_SUITE/include/libtest.shlib
@ -98,7 +99,6 @@ set -A badargs \
"-R $POOL/recvfs" \
"-p $POOL/recvfs" \
"-I $POOL/recvfs" \
"-D $POOL/recvfs" \
"-h $POOL/recvfs"
while (( i < ${#badargs[*]} ))