c6d9dc1ad2
* Illumos zfs issue #3035 [1] LZ4 compression support in ZFS. LZ4 is a new high-speed BSD-licensed compression algorithm created by Yann Collet that delivers very high compression and decompression performance compared to lzjb (>50% faster on compression, >80% faster on decompression and around 3x faster on compression of incompressible data), while giving better compression ratio [1]. This version of LZ4 corresponds to upstream's [2] revision 85. Please note that for obvious reasons this is not backward read compatible. This means once a pool have LZ4 compressed data, these data can no longer be read by older ZFS implementations. Local changes: - On-stack hash table disabled and using kernel slab allocator instead, at this time. This requires larger kernel thread stack for zio workers. This may change in the future should we adjusted the zio workers' thread stack size. - likely and unlikely will be undefined if they are already defined, this is required for i386 XEN build. - Removed De Bruijn sequence based __builtin_ctz family of builtins in favor of the latter. Both GCC and clang supports these builtins. - Changed the way the LZ4 code detects endianness. - Manual pages modifications to mention the feature based on Illumos counterpart. - Boot loader changes to make it support LZ4 decompression. [1] https://www.illumos.org/issues/3035 [2] http://code.google.com/p/lz4/source/list Obtained from: Illumos (13921:9d721847e469) Tested on: FreeBSD/amd64 MFC after: 1 month
1700 lines
48 KiB
C
1700 lines
48 KiB
C
/*
|
|
* CDDL HEADER START
|
|
*
|
|
* The contents of this file are subject to the terms of the
|
|
* Common Development and Distribution License (the "License").
|
|
* You may not use this file except in compliance with the License.
|
|
*
|
|
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
|
* or http://www.opensolaris.org/os/licensing.
|
|
* See the License for the specific language governing permissions
|
|
* and limitations under the License.
|
|
*
|
|
* When distributing Covered Code, include this CDDL HEADER in each
|
|
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
|
* If applicable, add the following below this CDDL HEADER, with the
|
|
* fields enclosed by brackets "[]" replaced with your own identifying
|
|
* information: Portions Copyright [yyyy] [name of copyright owner]
|
|
*
|
|
* CDDL HEADER END
|
|
*/
|
|
/*
|
|
* Copyright 2007 Sun Microsystems, Inc. All rights reserved.
|
|
* Use is subject to license terms.
|
|
*/
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
static uint64_t zfs_crc64_table[256];
|
|
|
|
#define ECKSUM 666
|
|
|
|
#define ASSERT(...) do { } while (0)
|
|
#define ASSERT3U(...) do { } while (0)
|
|
#define ASSERT3S(...) do { } while (0)
|
|
|
|
#define panic(...) do { \
|
|
printf(__VA_ARGS__); \
|
|
for (;;) ; \
|
|
} while (0)
|
|
|
|
#define kmem_alloc(size, flag) zfs_alloc((size))
|
|
#define kmem_free(ptr, size) zfs_free((ptr), (size))
|
|
|
|
static void
|
|
zfs_init_crc(void)
|
|
{
|
|
int i, j;
|
|
uint64_t *ct;
|
|
|
|
/*
|
|
* Calculate the crc64 table (used for the zap hash
|
|
* function).
|
|
*/
|
|
if (zfs_crc64_table[128] != ZFS_CRC64_POLY) {
|
|
memset(zfs_crc64_table, 0, sizeof(zfs_crc64_table));
|
|
for (i = 0; i < 256; i++)
|
|
for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
|
|
*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
|
|
}
|
|
}
|
|
|
|
static void
|
|
zio_checksum_off(const void *buf, uint64_t size, zio_cksum_t *zcp)
|
|
{
|
|
ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
|
|
}
|
|
|
|
/*
|
|
* Signature for checksum functions.
|
|
*/
|
|
typedef void zio_checksum_t(const void *data, uint64_t size, zio_cksum_t *zcp);
|
|
|
|
/*
|
|
* Information about each checksum function.
|
|
*/
|
|
typedef struct zio_checksum_info {
|
|
zio_checksum_t *ci_func[2]; /* checksum function for each byteorder */
|
|
int ci_correctable; /* number of correctable bits */
|
|
int ci_eck; /* uses zio embedded checksum? */
|
|
int ci_dedup; /* strong enough for dedup? */
|
|
const char *ci_name; /* descriptive name */
|
|
} zio_checksum_info_t;
|
|
|
|
#include "fletcher.c"
|
|
#include "sha256.c"
|
|
|
|
static zio_checksum_info_t zio_checksum_table[ZIO_CHECKSUM_FUNCTIONS] = {
|
|
{{NULL, NULL}, 0, 0, 0, "inherit"},
|
|
{{NULL, NULL}, 0, 0, 0, "on"},
|
|
{{zio_checksum_off, zio_checksum_off}, 0, 0, 0, "off"},
|
|
{{zio_checksum_SHA256, zio_checksum_SHA256}, 1, 1, 0, "label"},
|
|
{{zio_checksum_SHA256, zio_checksum_SHA256}, 1, 1, 0, "gang_header"},
|
|
{{fletcher_2_native, fletcher_2_byteswap}, 0, 1, 0, "zilog"},
|
|
{{fletcher_2_native, fletcher_2_byteswap}, 0, 0, 0, "fletcher2"},
|
|
{{fletcher_4_native, fletcher_4_byteswap}, 1, 0, 0, "fletcher4"},
|
|
{{zio_checksum_SHA256, zio_checksum_SHA256}, 1, 0, 1, "SHA256"},
|
|
{{fletcher_4_native, fletcher_4_byteswap}, 0, 1, 0, "zillog2"},
|
|
};
|
|
|
|
|
|
/*
|
|
* Common signature for all zio compress/decompress functions.
|
|
*/
|
|
typedef size_t zio_compress_func_t(void *src, void *dst,
|
|
size_t s_len, size_t d_len, int);
|
|
typedef int zio_decompress_func_t(void *src, void *dst,
|
|
size_t s_len, size_t d_len, int);
|
|
|
|
/*
|
|
* Information about each compression function.
|
|
*/
|
|
typedef struct zio_compress_info {
|
|
zio_compress_func_t *ci_compress; /* compression function */
|
|
zio_decompress_func_t *ci_decompress; /* decompression function */
|
|
int ci_level; /* level parameter */
|
|
const char *ci_name; /* algorithm name */
|
|
} zio_compress_info_t;
|
|
|
|
#include "lzjb.c"
|
|
#include "zle.c"
|
|
#include "lz4.c"
|
|
|
|
/*
|
|
* Compression vectors.
|
|
*/
|
|
static zio_compress_info_t zio_compress_table[ZIO_COMPRESS_FUNCTIONS] = {
|
|
{NULL, NULL, 0, "inherit"},
|
|
{NULL, NULL, 0, "on"},
|
|
{NULL, NULL, 0, "uncompressed"},
|
|
{NULL, lzjb_decompress, 0, "lzjb"},
|
|
{NULL, NULL, 0, "empty"},
|
|
{NULL, NULL, 1, "gzip-1"},
|
|
{NULL, NULL, 2, "gzip-2"},
|
|
{NULL, NULL, 3, "gzip-3"},
|
|
{NULL, NULL, 4, "gzip-4"},
|
|
{NULL, NULL, 5, "gzip-5"},
|
|
{NULL, NULL, 6, "gzip-6"},
|
|
{NULL, NULL, 7, "gzip-7"},
|
|
{NULL, NULL, 8, "gzip-8"},
|
|
{NULL, NULL, 9, "gzip-9"},
|
|
{NULL, zle_decompress, 64, "zle"},
|
|
{NULL, lz4_decompress, 0, "lz4"},
|
|
};
|
|
|
|
static void
|
|
byteswap_uint64_array(void *vbuf, size_t size)
|
|
{
|
|
uint64_t *buf = vbuf;
|
|
size_t count = size >> 3;
|
|
int i;
|
|
|
|
ASSERT((size & 7) == 0);
|
|
|
|
for (i = 0; i < count; i++)
|
|
buf[i] = BSWAP_64(buf[i]);
|
|
}
|
|
|
|
/*
|
|
* Set the external verifier for a gang block based on <vdev, offset, txg>,
|
|
* a tuple which is guaranteed to be unique for the life of the pool.
|
|
*/
|
|
static void
|
|
zio_checksum_gang_verifier(zio_cksum_t *zcp, const blkptr_t *bp)
|
|
{
|
|
const dva_t *dva = BP_IDENTITY(bp);
|
|
uint64_t txg = BP_PHYSICAL_BIRTH(bp);
|
|
|
|
ASSERT(BP_IS_GANG(bp));
|
|
|
|
ZIO_SET_CHECKSUM(zcp, DVA_GET_VDEV(dva), DVA_GET_OFFSET(dva), txg, 0);
|
|
}
|
|
|
|
/*
|
|
* Set the external verifier for a label block based on its offset.
|
|
* The vdev is implicit, and the txg is unknowable at pool open time --
|
|
* hence the logic in vdev_uberblock_load() to find the most recent copy.
|
|
*/
|
|
static void
|
|
zio_checksum_label_verifier(zio_cksum_t *zcp, uint64_t offset)
|
|
{
|
|
ZIO_SET_CHECKSUM(zcp, offset, 0, 0, 0);
|
|
}
|
|
|
|
static int
|
|
zio_checksum_verify(const blkptr_t *bp, void *data)
|
|
{
|
|
uint64_t size;
|
|
unsigned int checksum;
|
|
zio_checksum_info_t *ci;
|
|
zio_cksum_t actual_cksum, expected_cksum, verifier;
|
|
int byteswap;
|
|
|
|
checksum = BP_GET_CHECKSUM(bp);
|
|
size = BP_GET_PSIZE(bp);
|
|
|
|
if (checksum >= ZIO_CHECKSUM_FUNCTIONS)
|
|
return (EINVAL);
|
|
ci = &zio_checksum_table[checksum];
|
|
if (ci->ci_func[0] == NULL || ci->ci_func[1] == NULL)
|
|
return (EINVAL);
|
|
|
|
if (ci->ci_eck) {
|
|
zio_eck_t *eck;
|
|
|
|
ASSERT(checksum == ZIO_CHECKSUM_GANG_HEADER ||
|
|
checksum == ZIO_CHECKSUM_LABEL);
|
|
|
|
eck = (zio_eck_t *)((char *)data + size) - 1;
|
|
|
|
if (checksum == ZIO_CHECKSUM_GANG_HEADER)
|
|
zio_checksum_gang_verifier(&verifier, bp);
|
|
else if (checksum == ZIO_CHECKSUM_LABEL)
|
|
zio_checksum_label_verifier(&verifier,
|
|
DVA_GET_OFFSET(BP_IDENTITY(bp)));
|
|
else
|
|
verifier = bp->blk_cksum;
|
|
|
|
byteswap = (eck->zec_magic == BSWAP_64(ZEC_MAGIC));
|
|
|
|
if (byteswap)
|
|
byteswap_uint64_array(&verifier, sizeof (zio_cksum_t));
|
|
|
|
expected_cksum = eck->zec_cksum;
|
|
eck->zec_cksum = verifier;
|
|
ci->ci_func[byteswap](data, size, &actual_cksum);
|
|
eck->zec_cksum = expected_cksum;
|
|
|
|
if (byteswap)
|
|
byteswap_uint64_array(&expected_cksum,
|
|
sizeof (zio_cksum_t));
|
|
} else {
|
|
expected_cksum = bp->blk_cksum;
|
|
ci->ci_func[0](data, size, &actual_cksum);
|
|
}
|
|
|
|
if (!ZIO_CHECKSUM_EQUAL(actual_cksum, expected_cksum)) {
|
|
/*printf("ZFS: read checksum failed\n");*/
|
|
return (EIO);
|
|
}
|
|
|
|
return (0);
|
|
}
|
|
|
|
static int
|
|
zio_decompress_data(int cpfunc, void *src, uint64_t srcsize,
|
|
void *dest, uint64_t destsize)
|
|
{
|
|
zio_compress_info_t *ci;
|
|
|
|
if (cpfunc >= ZIO_COMPRESS_FUNCTIONS) {
|
|
printf("ZFS: unsupported compression algorithm %u\n", cpfunc);
|
|
return (EIO);
|
|
}
|
|
|
|
ci = &zio_compress_table[cpfunc];
|
|
if (!ci->ci_decompress) {
|
|
printf("ZFS: unsupported compression algorithm %s\n",
|
|
ci->ci_name);
|
|
return (EIO);
|
|
}
|
|
|
|
return (ci->ci_decompress(src, dest, srcsize, destsize, ci->ci_level));
|
|
}
|
|
|
|
static uint64_t
|
|
zap_hash(uint64_t salt, const char *name)
|
|
{
|
|
const uint8_t *cp;
|
|
uint8_t c;
|
|
uint64_t crc = salt;
|
|
|
|
ASSERT(crc != 0);
|
|
ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
|
|
for (cp = (const uint8_t *)name; (c = *cp) != '\0'; cp++)
|
|
crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ c) & 0xFF];
|
|
|
|
/*
|
|
* Only use 28 bits, since we need 4 bits in the cookie for the
|
|
* collision differentiator. We MUST use the high bits, since
|
|
* those are the onces that we first pay attention to when
|
|
* chosing the bucket.
|
|
*/
|
|
crc &= ~((1ULL << (64 - ZAP_HASHBITS)) - 1);
|
|
|
|
return (crc);
|
|
}
|
|
|
|
static void *zfs_alloc(size_t size);
|
|
static void zfs_free(void *ptr, size_t size);
|
|
|
|
typedef struct raidz_col {
|
|
uint64_t rc_devidx; /* child device index for I/O */
|
|
uint64_t rc_offset; /* device offset */
|
|
uint64_t rc_size; /* I/O size */
|
|
void *rc_data; /* I/O data */
|
|
int rc_error; /* I/O error for this device */
|
|
uint8_t rc_tried; /* Did we attempt this I/O column? */
|
|
uint8_t rc_skipped; /* Did we skip this I/O column? */
|
|
} raidz_col_t;
|
|
|
|
typedef struct raidz_map {
|
|
uint64_t rm_cols; /* Regular column count */
|
|
uint64_t rm_scols; /* Count including skipped columns */
|
|
uint64_t rm_bigcols; /* Number of oversized columns */
|
|
uint64_t rm_asize; /* Actual total I/O size */
|
|
uint64_t rm_missingdata; /* Count of missing data devices */
|
|
uint64_t rm_missingparity; /* Count of missing parity devices */
|
|
uint64_t rm_firstdatacol; /* First data column/parity count */
|
|
uint64_t rm_nskip; /* Skipped sectors for padding */
|
|
uint64_t rm_skipstart; /* Column index of padding start */
|
|
uintptr_t rm_reports; /* # of referencing checksum reports */
|
|
uint8_t rm_freed; /* map no longer has referencing ZIO */
|
|
uint8_t rm_ecksuminjected; /* checksum error was injected */
|
|
raidz_col_t rm_col[1]; /* Flexible array of I/O columns */
|
|
} raidz_map_t;
|
|
|
|
#define VDEV_RAIDZ_P 0
|
|
#define VDEV_RAIDZ_Q 1
|
|
#define VDEV_RAIDZ_R 2
|
|
|
|
#define VDEV_RAIDZ_MUL_2(x) (((x) << 1) ^ (((x) & 0x80) ? 0x1d : 0))
|
|
#define VDEV_RAIDZ_MUL_4(x) (VDEV_RAIDZ_MUL_2(VDEV_RAIDZ_MUL_2(x)))
|
|
|
|
/*
|
|
* We provide a mechanism to perform the field multiplication operation on a
|
|
* 64-bit value all at once rather than a byte at a time. This works by
|
|
* creating a mask from the top bit in each byte and using that to
|
|
* conditionally apply the XOR of 0x1d.
|
|
*/
|
|
#define VDEV_RAIDZ_64MUL_2(x, mask) \
|
|
{ \
|
|
(mask) = (x) & 0x8080808080808080ULL; \
|
|
(mask) = ((mask) << 1) - ((mask) >> 7); \
|
|
(x) = (((x) << 1) & 0xfefefefefefefefeULL) ^ \
|
|
((mask) & 0x1d1d1d1d1d1d1d1dULL); \
|
|
}
|
|
|
|
#define VDEV_RAIDZ_64MUL_4(x, mask) \
|
|
{ \
|
|
VDEV_RAIDZ_64MUL_2((x), mask); \
|
|
VDEV_RAIDZ_64MUL_2((x), mask); \
|
|
}
|
|
|
|
/*
|
|
* These two tables represent powers and logs of 2 in the Galois field defined
|
|
* above. These values were computed by repeatedly multiplying by 2 as above.
|
|
*/
|
|
static const uint8_t vdev_raidz_pow2[256] = {
|
|
0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
|
|
0x1d, 0x3a, 0x74, 0xe8, 0xcd, 0x87, 0x13, 0x26,
|
|
0x4c, 0x98, 0x2d, 0x5a, 0xb4, 0x75, 0xea, 0xc9,
|
|
0x8f, 0x03, 0x06, 0x0c, 0x18, 0x30, 0x60, 0xc0,
|
|
0x9d, 0x27, 0x4e, 0x9c, 0x25, 0x4a, 0x94, 0x35,
|
|
0x6a, 0xd4, 0xb5, 0x77, 0xee, 0xc1, 0x9f, 0x23,
|
|
0x46, 0x8c, 0x05, 0x0a, 0x14, 0x28, 0x50, 0xa0,
|
|
0x5d, 0xba, 0x69, 0xd2, 0xb9, 0x6f, 0xde, 0xa1,
|
|
0x5f, 0xbe, 0x61, 0xc2, 0x99, 0x2f, 0x5e, 0xbc,
|
|
0x65, 0xca, 0x89, 0x0f, 0x1e, 0x3c, 0x78, 0xf0,
|
|
0xfd, 0xe7, 0xd3, 0xbb, 0x6b, 0xd6, 0xb1, 0x7f,
|
|
0xfe, 0xe1, 0xdf, 0xa3, 0x5b, 0xb6, 0x71, 0xe2,
|
|
0xd9, 0xaf, 0x43, 0x86, 0x11, 0x22, 0x44, 0x88,
|
|
0x0d, 0x1a, 0x34, 0x68, 0xd0, 0xbd, 0x67, 0xce,
|
|
0x81, 0x1f, 0x3e, 0x7c, 0xf8, 0xed, 0xc7, 0x93,
|
|
0x3b, 0x76, 0xec, 0xc5, 0x97, 0x33, 0x66, 0xcc,
|
|
0x85, 0x17, 0x2e, 0x5c, 0xb8, 0x6d, 0xda, 0xa9,
|
|
0x4f, 0x9e, 0x21, 0x42, 0x84, 0x15, 0x2a, 0x54,
|
|
0xa8, 0x4d, 0x9a, 0x29, 0x52, 0xa4, 0x55, 0xaa,
|
|
0x49, 0x92, 0x39, 0x72, 0xe4, 0xd5, 0xb7, 0x73,
|
|
0xe6, 0xd1, 0xbf, 0x63, 0xc6, 0x91, 0x3f, 0x7e,
|
|
0xfc, 0xe5, 0xd7, 0xb3, 0x7b, 0xf6, 0xf1, 0xff,
|
|
0xe3, 0xdb, 0xab, 0x4b, 0x96, 0x31, 0x62, 0xc4,
|
|
0x95, 0x37, 0x6e, 0xdc, 0xa5, 0x57, 0xae, 0x41,
|
|
0x82, 0x19, 0x32, 0x64, 0xc8, 0x8d, 0x07, 0x0e,
|
|
0x1c, 0x38, 0x70, 0xe0, 0xdd, 0xa7, 0x53, 0xa6,
|
|
0x51, 0xa2, 0x59, 0xb2, 0x79, 0xf2, 0xf9, 0xef,
|
|
0xc3, 0x9b, 0x2b, 0x56, 0xac, 0x45, 0x8a, 0x09,
|
|
0x12, 0x24, 0x48, 0x90, 0x3d, 0x7a, 0xf4, 0xf5,
|
|
0xf7, 0xf3, 0xfb, 0xeb, 0xcb, 0x8b, 0x0b, 0x16,
|
|
0x2c, 0x58, 0xb0, 0x7d, 0xfa, 0xe9, 0xcf, 0x83,
|
|
0x1b, 0x36, 0x6c, 0xd8, 0xad, 0x47, 0x8e, 0x01
|
|
};
|
|
static const uint8_t vdev_raidz_log2[256] = {
|
|
0x00, 0x00, 0x01, 0x19, 0x02, 0x32, 0x1a, 0xc6,
|
|
0x03, 0xdf, 0x33, 0xee, 0x1b, 0x68, 0xc7, 0x4b,
|
|
0x04, 0x64, 0xe0, 0x0e, 0x34, 0x8d, 0xef, 0x81,
|
|
0x1c, 0xc1, 0x69, 0xf8, 0xc8, 0x08, 0x4c, 0x71,
|
|
0x05, 0x8a, 0x65, 0x2f, 0xe1, 0x24, 0x0f, 0x21,
|
|
0x35, 0x93, 0x8e, 0xda, 0xf0, 0x12, 0x82, 0x45,
|
|
0x1d, 0xb5, 0xc2, 0x7d, 0x6a, 0x27, 0xf9, 0xb9,
|
|
0xc9, 0x9a, 0x09, 0x78, 0x4d, 0xe4, 0x72, 0xa6,
|
|
0x06, 0xbf, 0x8b, 0x62, 0x66, 0xdd, 0x30, 0xfd,
|
|
0xe2, 0x98, 0x25, 0xb3, 0x10, 0x91, 0x22, 0x88,
|
|
0x36, 0xd0, 0x94, 0xce, 0x8f, 0x96, 0xdb, 0xbd,
|
|
0xf1, 0xd2, 0x13, 0x5c, 0x83, 0x38, 0x46, 0x40,
|
|
0x1e, 0x42, 0xb6, 0xa3, 0xc3, 0x48, 0x7e, 0x6e,
|
|
0x6b, 0x3a, 0x28, 0x54, 0xfa, 0x85, 0xba, 0x3d,
|
|
0xca, 0x5e, 0x9b, 0x9f, 0x0a, 0x15, 0x79, 0x2b,
|
|
0x4e, 0xd4, 0xe5, 0xac, 0x73, 0xf3, 0xa7, 0x57,
|
|
0x07, 0x70, 0xc0, 0xf7, 0x8c, 0x80, 0x63, 0x0d,
|
|
0x67, 0x4a, 0xde, 0xed, 0x31, 0xc5, 0xfe, 0x18,
|
|
0xe3, 0xa5, 0x99, 0x77, 0x26, 0xb8, 0xb4, 0x7c,
|
|
0x11, 0x44, 0x92, 0xd9, 0x23, 0x20, 0x89, 0x2e,
|
|
0x37, 0x3f, 0xd1, 0x5b, 0x95, 0xbc, 0xcf, 0xcd,
|
|
0x90, 0x87, 0x97, 0xb2, 0xdc, 0xfc, 0xbe, 0x61,
|
|
0xf2, 0x56, 0xd3, 0xab, 0x14, 0x2a, 0x5d, 0x9e,
|
|
0x84, 0x3c, 0x39, 0x53, 0x47, 0x6d, 0x41, 0xa2,
|
|
0x1f, 0x2d, 0x43, 0xd8, 0xb7, 0x7b, 0xa4, 0x76,
|
|
0xc4, 0x17, 0x49, 0xec, 0x7f, 0x0c, 0x6f, 0xf6,
|
|
0x6c, 0xa1, 0x3b, 0x52, 0x29, 0x9d, 0x55, 0xaa,
|
|
0xfb, 0x60, 0x86, 0xb1, 0xbb, 0xcc, 0x3e, 0x5a,
|
|
0xcb, 0x59, 0x5f, 0xb0, 0x9c, 0xa9, 0xa0, 0x51,
|
|
0x0b, 0xf5, 0x16, 0xeb, 0x7a, 0x75, 0x2c, 0xd7,
|
|
0x4f, 0xae, 0xd5, 0xe9, 0xe6, 0xe7, 0xad, 0xe8,
|
|
0x74, 0xd6, 0xf4, 0xea, 0xa8, 0x50, 0x58, 0xaf,
|
|
};
|
|
|
|
/*
|
|
* Multiply a given number by 2 raised to the given power.
|
|
*/
|
|
static uint8_t
|
|
vdev_raidz_exp2(uint8_t a, int exp)
|
|
{
|
|
if (a == 0)
|
|
return (0);
|
|
|
|
ASSERT(exp >= 0);
|
|
ASSERT(vdev_raidz_log2[a] > 0 || a == 1);
|
|
|
|
exp += vdev_raidz_log2[a];
|
|
if (exp > 255)
|
|
exp -= 255;
|
|
|
|
return (vdev_raidz_pow2[exp]);
|
|
}
|
|
|
|
static void
|
|
vdev_raidz_generate_parity_p(raidz_map_t *rm)
|
|
{
|
|
uint64_t *p, *src, pcount, ccount, i;
|
|
int c;
|
|
|
|
pcount = rm->rm_col[VDEV_RAIDZ_P].rc_size / sizeof (src[0]);
|
|
|
|
for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
|
|
src = rm->rm_col[c].rc_data;
|
|
p = rm->rm_col[VDEV_RAIDZ_P].rc_data;
|
|
ccount = rm->rm_col[c].rc_size / sizeof (src[0]);
|
|
|
|
if (c == rm->rm_firstdatacol) {
|
|
ASSERT(ccount == pcount);
|
|
for (i = 0; i < ccount; i++, src++, p++) {
|
|
*p = *src;
|
|
}
|
|
} else {
|
|
ASSERT(ccount <= pcount);
|
|
for (i = 0; i < ccount; i++, src++, p++) {
|
|
*p ^= *src;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
vdev_raidz_generate_parity_pq(raidz_map_t *rm)
|
|
{
|
|
uint64_t *p, *q, *src, pcnt, ccnt, mask, i;
|
|
int c;
|
|
|
|
pcnt = rm->rm_col[VDEV_RAIDZ_P].rc_size / sizeof (src[0]);
|
|
ASSERT(rm->rm_col[VDEV_RAIDZ_P].rc_size ==
|
|
rm->rm_col[VDEV_RAIDZ_Q].rc_size);
|
|
|
|
for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
|
|
src = rm->rm_col[c].rc_data;
|
|
p = rm->rm_col[VDEV_RAIDZ_P].rc_data;
|
|
q = rm->rm_col[VDEV_RAIDZ_Q].rc_data;
|
|
|
|
ccnt = rm->rm_col[c].rc_size / sizeof (src[0]);
|
|
|
|
if (c == rm->rm_firstdatacol) {
|
|
ASSERT(ccnt == pcnt || ccnt == 0);
|
|
for (i = 0; i < ccnt; i++, src++, p++, q++) {
|
|
*p = *src;
|
|
*q = *src;
|
|
}
|
|
for (; i < pcnt; i++, src++, p++, q++) {
|
|
*p = 0;
|
|
*q = 0;
|
|
}
|
|
} else {
|
|
ASSERT(ccnt <= pcnt);
|
|
|
|
/*
|
|
* Apply the algorithm described above by multiplying
|
|
* the previous result and adding in the new value.
|
|
*/
|
|
for (i = 0; i < ccnt; i++, src++, p++, q++) {
|
|
*p ^= *src;
|
|
|
|
VDEV_RAIDZ_64MUL_2(*q, mask);
|
|
*q ^= *src;
|
|
}
|
|
|
|
/*
|
|
* Treat short columns as though they are full of 0s.
|
|
* Note that there's therefore nothing needed for P.
|
|
*/
|
|
for (; i < pcnt; i++, q++) {
|
|
VDEV_RAIDZ_64MUL_2(*q, mask);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
vdev_raidz_generate_parity_pqr(raidz_map_t *rm)
|
|
{
|
|
uint64_t *p, *q, *r, *src, pcnt, ccnt, mask, i;
|
|
int c;
|
|
|
|
pcnt = rm->rm_col[VDEV_RAIDZ_P].rc_size / sizeof (src[0]);
|
|
ASSERT(rm->rm_col[VDEV_RAIDZ_P].rc_size ==
|
|
rm->rm_col[VDEV_RAIDZ_Q].rc_size);
|
|
ASSERT(rm->rm_col[VDEV_RAIDZ_P].rc_size ==
|
|
rm->rm_col[VDEV_RAIDZ_R].rc_size);
|
|
|
|
for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
|
|
src = rm->rm_col[c].rc_data;
|
|
p = rm->rm_col[VDEV_RAIDZ_P].rc_data;
|
|
q = rm->rm_col[VDEV_RAIDZ_Q].rc_data;
|
|
r = rm->rm_col[VDEV_RAIDZ_R].rc_data;
|
|
|
|
ccnt = rm->rm_col[c].rc_size / sizeof (src[0]);
|
|
|
|
if (c == rm->rm_firstdatacol) {
|
|
ASSERT(ccnt == pcnt || ccnt == 0);
|
|
for (i = 0; i < ccnt; i++, src++, p++, q++, r++) {
|
|
*p = *src;
|
|
*q = *src;
|
|
*r = *src;
|
|
}
|
|
for (; i < pcnt; i++, src++, p++, q++, r++) {
|
|
*p = 0;
|
|
*q = 0;
|
|
*r = 0;
|
|
}
|
|
} else {
|
|
ASSERT(ccnt <= pcnt);
|
|
|
|
/*
|
|
* Apply the algorithm described above by multiplying
|
|
* the previous result and adding in the new value.
|
|
*/
|
|
for (i = 0; i < ccnt; i++, src++, p++, q++, r++) {
|
|
*p ^= *src;
|
|
|
|
VDEV_RAIDZ_64MUL_2(*q, mask);
|
|
*q ^= *src;
|
|
|
|
VDEV_RAIDZ_64MUL_4(*r, mask);
|
|
*r ^= *src;
|
|
}
|
|
|
|
/*
|
|
* Treat short columns as though they are full of 0s.
|
|
* Note that there's therefore nothing needed for P.
|
|
*/
|
|
for (; i < pcnt; i++, q++, r++) {
|
|
VDEV_RAIDZ_64MUL_2(*q, mask);
|
|
VDEV_RAIDZ_64MUL_4(*r, mask);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Generate RAID parity in the first virtual columns according to the number of
|
|
* parity columns available.
|
|
*/
|
|
static void
|
|
vdev_raidz_generate_parity(raidz_map_t *rm)
|
|
{
|
|
switch (rm->rm_firstdatacol) {
|
|
case 1:
|
|
vdev_raidz_generate_parity_p(rm);
|
|
break;
|
|
case 2:
|
|
vdev_raidz_generate_parity_pq(rm);
|
|
break;
|
|
case 3:
|
|
vdev_raidz_generate_parity_pqr(rm);
|
|
break;
|
|
default:
|
|
panic("invalid RAID-Z configuration");
|
|
}
|
|
}
|
|
|
|
/* BEGIN CSTYLED */
|
|
/*
|
|
* In the general case of reconstruction, we must solve the system of linear
|
|
* equations defined by the coeffecients used to generate parity as well as
|
|
* the contents of the data and parity disks. This can be expressed with
|
|
* vectors for the original data (D) and the actual data (d) and parity (p)
|
|
* and a matrix composed of the identity matrix (I) and a dispersal matrix (V):
|
|
*
|
|
* __ __ __ __
|
|
* | | __ __ | p_0 |
|
|
* | V | | D_0 | | p_m-1 |
|
|
* | | x | : | = | d_0 |
|
|
* | I | | D_n-1 | | : |
|
|
* | | ~~ ~~ | d_n-1 |
|
|
* ~~ ~~ ~~ ~~
|
|
*
|
|
* I is simply a square identity matrix of size n, and V is a vandermonde
|
|
* matrix defined by the coeffecients we chose for the various parity columns
|
|
* (1, 2, 4). Note that these values were chosen both for simplicity, speedy
|
|
* computation as well as linear separability.
|
|
*
|
|
* __ __ __ __
|
|
* | 1 .. 1 1 1 | | p_0 |
|
|
* | 2^n-1 .. 4 2 1 | __ __ | : |
|
|
* | 4^n-1 .. 16 4 1 | | D_0 | | p_m-1 |
|
|
* | 1 .. 0 0 0 | | D_1 | | d_0 |
|
|
* | 0 .. 0 0 0 | x | D_2 | = | d_1 |
|
|
* | : : : : | | : | | d_2 |
|
|
* | 0 .. 1 0 0 | | D_n-1 | | : |
|
|
* | 0 .. 0 1 0 | ~~ ~~ | : |
|
|
* | 0 .. 0 0 1 | | d_n-1 |
|
|
* ~~ ~~ ~~ ~~
|
|
*
|
|
* Note that I, V, d, and p are known. To compute D, we must invert the
|
|
* matrix and use the known data and parity values to reconstruct the unknown
|
|
* data values. We begin by removing the rows in V|I and d|p that correspond
|
|
* to failed or missing columns; we then make V|I square (n x n) and d|p
|
|
* sized n by removing rows corresponding to unused parity from the bottom up
|
|
* to generate (V|I)' and (d|p)'. We can then generate the inverse of (V|I)'
|
|
* using Gauss-Jordan elimination. In the example below we use m=3 parity
|
|
* columns, n=8 data columns, with errors in d_1, d_2, and p_1:
|
|
* __ __
|
|
* | 1 1 1 1 1 1 1 1 |
|
|
* | 128 64 32 16 8 4 2 1 | <-----+-+-- missing disks
|
|
* | 19 205 116 29 64 16 4 1 | / /
|
|
* | 1 0 0 0 0 0 0 0 | / /
|
|
* | 0 1 0 0 0 0 0 0 | <--' /
|
|
* (V|I) = | 0 0 1 0 0 0 0 0 | <---'
|
|
* | 0 0 0 1 0 0 0 0 |
|
|
* | 0 0 0 0 1 0 0 0 |
|
|
* | 0 0 0 0 0 1 0 0 |
|
|
* | 0 0 0 0 0 0 1 0 |
|
|
* | 0 0 0 0 0 0 0 1 |
|
|
* ~~ ~~
|
|
* __ __
|
|
* | 1 1 1 1 1 1 1 1 |
|
|
* | 128 64 32 16 8 4 2 1 |
|
|
* | 19 205 116 29 64 16 4 1 |
|
|
* | 1 0 0 0 0 0 0 0 |
|
|
* | 0 1 0 0 0 0 0 0 |
|
|
* (V|I)' = | 0 0 1 0 0 0 0 0 |
|
|
* | 0 0 0 1 0 0 0 0 |
|
|
* | 0 0 0 0 1 0 0 0 |
|
|
* | 0 0 0 0 0 1 0 0 |
|
|
* | 0 0 0 0 0 0 1 0 |
|
|
* | 0 0 0 0 0 0 0 1 |
|
|
* ~~ ~~
|
|
*
|
|
* Here we employ Gauss-Jordan elimination to find the inverse of (V|I)'. We
|
|
* have carefully chosen the seed values 1, 2, and 4 to ensure that this
|
|
* matrix is not singular.
|
|
* __ __
|
|
* | 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 |
|
|
* | 19 205 116 29 64 16 4 1 0 1 0 0 0 0 0 0 |
|
|
* | 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 |
|
|
* | 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 |
|
|
* | 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 |
|
|
* | 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 |
|
|
* | 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 |
|
|
* | 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 |
|
|
* ~~ ~~
|
|
* __ __
|
|
* | 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 |
|
|
* | 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 |
|
|
* | 19 205 116 29 64 16 4 1 0 1 0 0 0 0 0 0 |
|
|
* | 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 |
|
|
* | 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 |
|
|
* | 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 |
|
|
* | 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 |
|
|
* | 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 |
|
|
* ~~ ~~
|
|
* __ __
|
|
* | 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 |
|
|
* | 0 1 1 0 0 0 0 0 1 0 1 1 1 1 1 1 |
|
|
* | 0 205 116 0 0 0 0 0 0 1 19 29 64 16 4 1 |
|
|
* | 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 |
|
|
* | 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 |
|
|
* | 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 |
|
|
* | 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 |
|
|
* | 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 |
|
|
* ~~ ~~
|
|
* __ __
|
|
* | 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 |
|
|
* | 0 1 1 0 0 0 0 0 1 0 1 1 1 1 1 1 |
|
|
* | 0 0 185 0 0 0 0 0 205 1 222 208 141 221 201 204 |
|
|
* | 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 |
|
|
* | 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 |
|
|
* | 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 |
|
|
* | 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 |
|
|
* | 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 |
|
|
* ~~ ~~
|
|
* __ __
|
|
* | 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 |
|
|
* | 0 1 1 0 0 0 0 0 1 0 1 1 1 1 1 1 |
|
|
* | 0 0 1 0 0 0 0 0 166 100 4 40 158 168 216 209 |
|
|
* | 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 |
|
|
* | 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 |
|
|
* | 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 |
|
|
* | 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 |
|
|
* | 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 |
|
|
* ~~ ~~
|
|
* __ __
|
|
* | 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 |
|
|
* | 0 1 0 0 0 0 0 0 167 100 5 41 159 169 217 208 |
|
|
* | 0 0 1 0 0 0 0 0 166 100 4 40 158 168 216 209 |
|
|
* | 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 |
|
|
* | 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 |
|
|
* | 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 |
|
|
* | 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 |
|
|
* | 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 |
|
|
* ~~ ~~
|
|
* __ __
|
|
* | 0 0 1 0 0 0 0 0 |
|
|
* | 167 100 5 41 159 169 217 208 |
|
|
* | 166 100 4 40 158 168 216 209 |
|
|
* (V|I)'^-1 = | 0 0 0 1 0 0 0 0 |
|
|
* | 0 0 0 0 1 0 0 0 |
|
|
* | 0 0 0 0 0 1 0 0 |
|
|
* | 0 0 0 0 0 0 1 0 |
|
|
* | 0 0 0 0 0 0 0 1 |
|
|
* ~~ ~~
|
|
*
|
|
* We can then simply compute D = (V|I)'^-1 x (d|p)' to discover the values
|
|
* of the missing data.
|
|
*
|
|
* As is apparent from the example above, the only non-trivial rows in the
|
|
* inverse matrix correspond to the data disks that we're trying to
|
|
* reconstruct. Indeed, those are the only rows we need as the others would
|
|
* only be useful for reconstructing data known or assumed to be valid. For
|
|
* that reason, we only build the coefficients in the rows that correspond to
|
|
* targeted columns.
|
|
*/
|
|
/* END CSTYLED */
|
|
|
|
static void
|
|
vdev_raidz_matrix_init(raidz_map_t *rm, int n, int nmap, int *map,
|
|
uint8_t **rows)
|
|
{
|
|
int i, j;
|
|
int pow;
|
|
|
|
ASSERT(n == rm->rm_cols - rm->rm_firstdatacol);
|
|
|
|
/*
|
|
* Fill in the missing rows of interest.
|
|
*/
|
|
for (i = 0; i < nmap; i++) {
|
|
ASSERT3S(0, <=, map[i]);
|
|
ASSERT3S(map[i], <=, 2);
|
|
|
|
pow = map[i] * n;
|
|
if (pow > 255)
|
|
pow -= 255;
|
|
ASSERT(pow <= 255);
|
|
|
|
for (j = 0; j < n; j++) {
|
|
pow -= map[i];
|
|
if (pow < 0)
|
|
pow += 255;
|
|
rows[i][j] = vdev_raidz_pow2[pow];
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
vdev_raidz_matrix_invert(raidz_map_t *rm, int n, int nmissing, int *missing,
|
|
uint8_t **rows, uint8_t **invrows, const uint8_t *used)
|
|
{
|
|
int i, j, ii, jj;
|
|
uint8_t log;
|
|
|
|
/*
|
|
* Assert that the first nmissing entries from the array of used
|
|
* columns correspond to parity columns and that subsequent entries
|
|
* correspond to data columns.
|
|
*/
|
|
for (i = 0; i < nmissing; i++) {
|
|
ASSERT3S(used[i], <, rm->rm_firstdatacol);
|
|
}
|
|
for (; i < n; i++) {
|
|
ASSERT3S(used[i], >=, rm->rm_firstdatacol);
|
|
}
|
|
|
|
/*
|
|
* First initialize the storage where we'll compute the inverse rows.
|
|
*/
|
|
for (i = 0; i < nmissing; i++) {
|
|
for (j = 0; j < n; j++) {
|
|
invrows[i][j] = (i == j) ? 1 : 0;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Subtract all trivial rows from the rows of consequence.
|
|
*/
|
|
for (i = 0; i < nmissing; i++) {
|
|
for (j = nmissing; j < n; j++) {
|
|
ASSERT3U(used[j], >=, rm->rm_firstdatacol);
|
|
jj = used[j] - rm->rm_firstdatacol;
|
|
ASSERT3S(jj, <, n);
|
|
invrows[i][j] = rows[i][jj];
|
|
rows[i][jj] = 0;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* For each of the rows of interest, we must normalize it and subtract
|
|
* a multiple of it from the other rows.
|
|
*/
|
|
for (i = 0; i < nmissing; i++) {
|
|
for (j = 0; j < missing[i]; j++) {
|
|
ASSERT3U(rows[i][j], ==, 0);
|
|
}
|
|
ASSERT3U(rows[i][missing[i]], !=, 0);
|
|
|
|
/*
|
|
* Compute the inverse of the first element and multiply each
|
|
* element in the row by that value.
|
|
*/
|
|
log = 255 - vdev_raidz_log2[rows[i][missing[i]]];
|
|
|
|
for (j = 0; j < n; j++) {
|
|
rows[i][j] = vdev_raidz_exp2(rows[i][j], log);
|
|
invrows[i][j] = vdev_raidz_exp2(invrows[i][j], log);
|
|
}
|
|
|
|
for (ii = 0; ii < nmissing; ii++) {
|
|
if (i == ii)
|
|
continue;
|
|
|
|
ASSERT3U(rows[ii][missing[i]], !=, 0);
|
|
|
|
log = vdev_raidz_log2[rows[ii][missing[i]]];
|
|
|
|
for (j = 0; j < n; j++) {
|
|
rows[ii][j] ^=
|
|
vdev_raidz_exp2(rows[i][j], log);
|
|
invrows[ii][j] ^=
|
|
vdev_raidz_exp2(invrows[i][j], log);
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Verify that the data that is left in the rows are properly part of
|
|
* an identity matrix.
|
|
*/
|
|
for (i = 0; i < nmissing; i++) {
|
|
for (j = 0; j < n; j++) {
|
|
if (j == missing[i]) {
|
|
ASSERT3U(rows[i][j], ==, 1);
|
|
} else {
|
|
ASSERT3U(rows[i][j], ==, 0);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
vdev_raidz_matrix_reconstruct(raidz_map_t *rm, int n, int nmissing,
|
|
int *missing, uint8_t **invrows, const uint8_t *used)
|
|
{
|
|
int i, j, x, cc, c;
|
|
uint8_t *src;
|
|
uint64_t ccount;
|
|
uint8_t *dst[VDEV_RAIDZ_MAXPARITY];
|
|
uint64_t dcount[VDEV_RAIDZ_MAXPARITY];
|
|
uint8_t log, val;
|
|
int ll;
|
|
uint8_t *invlog[VDEV_RAIDZ_MAXPARITY];
|
|
uint8_t *p, *pp;
|
|
size_t psize;
|
|
|
|
log = 0; /* gcc */
|
|
psize = sizeof (invlog[0][0]) * n * nmissing;
|
|
p = zfs_alloc(psize);
|
|
|
|
for (pp = p, i = 0; i < nmissing; i++) {
|
|
invlog[i] = pp;
|
|
pp += n;
|
|
}
|
|
|
|
for (i = 0; i < nmissing; i++) {
|
|
for (j = 0; j < n; j++) {
|
|
ASSERT3U(invrows[i][j], !=, 0);
|
|
invlog[i][j] = vdev_raidz_log2[invrows[i][j]];
|
|
}
|
|
}
|
|
|
|
for (i = 0; i < n; i++) {
|
|
c = used[i];
|
|
ASSERT3U(c, <, rm->rm_cols);
|
|
|
|
src = rm->rm_col[c].rc_data;
|
|
ccount = rm->rm_col[c].rc_size;
|
|
for (j = 0; j < nmissing; j++) {
|
|
cc = missing[j] + rm->rm_firstdatacol;
|
|
ASSERT3U(cc, >=, rm->rm_firstdatacol);
|
|
ASSERT3U(cc, <, rm->rm_cols);
|
|
ASSERT3U(cc, !=, c);
|
|
|
|
dst[j] = rm->rm_col[cc].rc_data;
|
|
dcount[j] = rm->rm_col[cc].rc_size;
|
|
}
|
|
|
|
ASSERT(ccount >= rm->rm_col[missing[0]].rc_size || i > 0);
|
|
|
|
for (x = 0; x < ccount; x++, src++) {
|
|
if (*src != 0)
|
|
log = vdev_raidz_log2[*src];
|
|
|
|
for (cc = 0; cc < nmissing; cc++) {
|
|
if (x >= dcount[cc])
|
|
continue;
|
|
|
|
if (*src == 0) {
|
|
val = 0;
|
|
} else {
|
|
if ((ll = log + invlog[cc][i]) >= 255)
|
|
ll -= 255;
|
|
val = vdev_raidz_pow2[ll];
|
|
}
|
|
|
|
if (i == 0)
|
|
dst[cc][x] = val;
|
|
else
|
|
dst[cc][x] ^= val;
|
|
}
|
|
}
|
|
}
|
|
|
|
zfs_free(p, psize);
|
|
}
|
|
|
|
static int
|
|
vdev_raidz_reconstruct_general(raidz_map_t *rm, int *tgts, int ntgts)
|
|
{
|
|
int n, i, c, t, tt;
|
|
int nmissing_rows;
|
|
int missing_rows[VDEV_RAIDZ_MAXPARITY];
|
|
int parity_map[VDEV_RAIDZ_MAXPARITY];
|
|
|
|
uint8_t *p, *pp;
|
|
size_t psize;
|
|
|
|
uint8_t *rows[VDEV_RAIDZ_MAXPARITY];
|
|
uint8_t *invrows[VDEV_RAIDZ_MAXPARITY];
|
|
uint8_t *used;
|
|
|
|
int code = 0;
|
|
|
|
|
|
n = rm->rm_cols - rm->rm_firstdatacol;
|
|
|
|
/*
|
|
* Figure out which data columns are missing.
|
|
*/
|
|
nmissing_rows = 0;
|
|
for (t = 0; t < ntgts; t++) {
|
|
if (tgts[t] >= rm->rm_firstdatacol) {
|
|
missing_rows[nmissing_rows++] =
|
|
tgts[t] - rm->rm_firstdatacol;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Figure out which parity columns to use to help generate the missing
|
|
* data columns.
|
|
*/
|
|
for (tt = 0, c = 0, i = 0; i < nmissing_rows; c++) {
|
|
ASSERT(tt < ntgts);
|
|
ASSERT(c < rm->rm_firstdatacol);
|
|
|
|
/*
|
|
* Skip any targeted parity columns.
|
|
*/
|
|
if (c == tgts[tt]) {
|
|
tt++;
|
|
continue;
|
|
}
|
|
|
|
code |= 1 << c;
|
|
|
|
parity_map[i] = c;
|
|
i++;
|
|
}
|
|
|
|
ASSERT(code != 0);
|
|
ASSERT3U(code, <, 1 << VDEV_RAIDZ_MAXPARITY);
|
|
|
|
psize = (sizeof (rows[0][0]) + sizeof (invrows[0][0])) *
|
|
nmissing_rows * n + sizeof (used[0]) * n;
|
|
p = kmem_alloc(psize, KM_SLEEP);
|
|
|
|
for (pp = p, i = 0; i < nmissing_rows; i++) {
|
|
rows[i] = pp;
|
|
pp += n;
|
|
invrows[i] = pp;
|
|
pp += n;
|
|
}
|
|
used = pp;
|
|
|
|
for (i = 0; i < nmissing_rows; i++) {
|
|
used[i] = parity_map[i];
|
|
}
|
|
|
|
for (tt = 0, c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
|
|
if (tt < nmissing_rows &&
|
|
c == missing_rows[tt] + rm->rm_firstdatacol) {
|
|
tt++;
|
|
continue;
|
|
}
|
|
|
|
ASSERT3S(i, <, n);
|
|
used[i] = c;
|
|
i++;
|
|
}
|
|
|
|
/*
|
|
* Initialize the interesting rows of the matrix.
|
|
*/
|
|
vdev_raidz_matrix_init(rm, n, nmissing_rows, parity_map, rows);
|
|
|
|
/*
|
|
* Invert the matrix.
|
|
*/
|
|
vdev_raidz_matrix_invert(rm, n, nmissing_rows, missing_rows, rows,
|
|
invrows, used);
|
|
|
|
/*
|
|
* Reconstruct the missing data using the generated matrix.
|
|
*/
|
|
vdev_raidz_matrix_reconstruct(rm, n, nmissing_rows, missing_rows,
|
|
invrows, used);
|
|
|
|
kmem_free(p, psize);
|
|
|
|
return (code);
|
|
}
|
|
|
|
static int
|
|
vdev_raidz_reconstruct(raidz_map_t *rm, int *t, int nt)
|
|
{
|
|
int tgts[VDEV_RAIDZ_MAXPARITY];
|
|
int ntgts;
|
|
int i, c;
|
|
int code;
|
|
int nbadparity, nbaddata;
|
|
|
|
/*
|
|
* The tgts list must already be sorted.
|
|
*/
|
|
for (i = 1; i < nt; i++) {
|
|
ASSERT(t[i] > t[i - 1]);
|
|
}
|
|
|
|
nbadparity = rm->rm_firstdatacol;
|
|
nbaddata = rm->rm_cols - nbadparity;
|
|
ntgts = 0;
|
|
for (i = 0, c = 0; c < rm->rm_cols; c++) {
|
|
if (i < nt && c == t[i]) {
|
|
tgts[ntgts++] = c;
|
|
i++;
|
|
} else if (rm->rm_col[c].rc_error != 0) {
|
|
tgts[ntgts++] = c;
|
|
} else if (c >= rm->rm_firstdatacol) {
|
|
nbaddata--;
|
|
} else {
|
|
nbadparity--;
|
|
}
|
|
}
|
|
|
|
ASSERT(ntgts >= nt);
|
|
ASSERT(nbaddata >= 0);
|
|
ASSERT(nbaddata + nbadparity == ntgts);
|
|
|
|
code = vdev_raidz_reconstruct_general(rm, tgts, ntgts);
|
|
ASSERT(code < (1 << VDEV_RAIDZ_MAXPARITY));
|
|
ASSERT(code > 0);
|
|
return (code);
|
|
}
|
|
|
|
static raidz_map_t *
|
|
vdev_raidz_map_alloc(void *data, off_t offset, size_t size, uint64_t unit_shift,
|
|
uint64_t dcols, uint64_t nparity)
|
|
{
|
|
raidz_map_t *rm;
|
|
uint64_t b = offset >> unit_shift;
|
|
uint64_t s = size >> unit_shift;
|
|
uint64_t f = b % dcols;
|
|
uint64_t o = (b / dcols) << unit_shift;
|
|
uint64_t q, r, c, bc, col, acols, scols, coff, devidx, asize, tot;
|
|
|
|
q = s / (dcols - nparity);
|
|
r = s - q * (dcols - nparity);
|
|
bc = (r == 0 ? 0 : r + nparity);
|
|
tot = s + nparity * (q + (r == 0 ? 0 : 1));
|
|
|
|
if (q == 0) {
|
|
acols = bc;
|
|
scols = MIN(dcols, roundup(bc, nparity + 1));
|
|
} else {
|
|
acols = dcols;
|
|
scols = dcols;
|
|
}
|
|
|
|
ASSERT3U(acols, <=, scols);
|
|
|
|
rm = zfs_alloc(offsetof(raidz_map_t, rm_col[scols]));
|
|
|
|
rm->rm_cols = acols;
|
|
rm->rm_scols = scols;
|
|
rm->rm_bigcols = bc;
|
|
rm->rm_skipstart = bc;
|
|
rm->rm_missingdata = 0;
|
|
rm->rm_missingparity = 0;
|
|
rm->rm_firstdatacol = nparity;
|
|
rm->rm_reports = 0;
|
|
rm->rm_freed = 0;
|
|
rm->rm_ecksuminjected = 0;
|
|
|
|
asize = 0;
|
|
|
|
for (c = 0; c < scols; c++) {
|
|
col = f + c;
|
|
coff = o;
|
|
if (col >= dcols) {
|
|
col -= dcols;
|
|
coff += 1ULL << unit_shift;
|
|
}
|
|
rm->rm_col[c].rc_devidx = col;
|
|
rm->rm_col[c].rc_offset = coff;
|
|
rm->rm_col[c].rc_data = NULL;
|
|
rm->rm_col[c].rc_error = 0;
|
|
rm->rm_col[c].rc_tried = 0;
|
|
rm->rm_col[c].rc_skipped = 0;
|
|
|
|
if (c >= acols)
|
|
rm->rm_col[c].rc_size = 0;
|
|
else if (c < bc)
|
|
rm->rm_col[c].rc_size = (q + 1) << unit_shift;
|
|
else
|
|
rm->rm_col[c].rc_size = q << unit_shift;
|
|
|
|
asize += rm->rm_col[c].rc_size;
|
|
}
|
|
|
|
ASSERT3U(asize, ==, tot << unit_shift);
|
|
rm->rm_asize = roundup(asize, (nparity + 1) << unit_shift);
|
|
rm->rm_nskip = roundup(tot, nparity + 1) - tot;
|
|
ASSERT3U(rm->rm_asize - asize, ==, rm->rm_nskip << unit_shift);
|
|
ASSERT3U(rm->rm_nskip, <=, nparity);
|
|
|
|
for (c = 0; c < rm->rm_firstdatacol; c++)
|
|
rm->rm_col[c].rc_data = zfs_alloc(rm->rm_col[c].rc_size);
|
|
|
|
rm->rm_col[c].rc_data = data;
|
|
|
|
for (c = c + 1; c < acols; c++)
|
|
rm->rm_col[c].rc_data = (char *)rm->rm_col[c - 1].rc_data +
|
|
rm->rm_col[c - 1].rc_size;
|
|
|
|
/*
|
|
* If all data stored spans all columns, there's a danger that parity
|
|
* will always be on the same device and, since parity isn't read
|
|
* during normal operation, that that device's I/O bandwidth won't be
|
|
* used effectively. We therefore switch the parity every 1MB.
|
|
*
|
|
* ... at least that was, ostensibly, the theory. As a practical
|
|
* matter unless we juggle the parity between all devices evenly, we
|
|
* won't see any benefit. Further, occasional writes that aren't a
|
|
* multiple of the LCM of the number of children and the minimum
|
|
* stripe width are sufficient to avoid pessimal behavior.
|
|
* Unfortunately, this decision created an implicit on-disk format
|
|
* requirement that we need to support for all eternity, but only
|
|
* for single-parity RAID-Z.
|
|
*
|
|
* If we intend to skip a sector in the zeroth column for padding
|
|
* we must make sure to note this swap. We will never intend to
|
|
* skip the first column since at least one data and one parity
|
|
* column must appear in each row.
|
|
*/
|
|
ASSERT(rm->rm_cols >= 2);
|
|
ASSERT(rm->rm_col[0].rc_size == rm->rm_col[1].rc_size);
|
|
|
|
if (rm->rm_firstdatacol == 1 && (offset & (1ULL << 20))) {
|
|
devidx = rm->rm_col[0].rc_devidx;
|
|
o = rm->rm_col[0].rc_offset;
|
|
rm->rm_col[0].rc_devidx = rm->rm_col[1].rc_devidx;
|
|
rm->rm_col[0].rc_offset = rm->rm_col[1].rc_offset;
|
|
rm->rm_col[1].rc_devidx = devidx;
|
|
rm->rm_col[1].rc_offset = o;
|
|
|
|
if (rm->rm_skipstart == 0)
|
|
rm->rm_skipstart = 1;
|
|
}
|
|
|
|
return (rm);
|
|
}
|
|
|
|
static void
|
|
vdev_raidz_map_free(raidz_map_t *rm)
|
|
{
|
|
int c;
|
|
|
|
for (c = rm->rm_firstdatacol - 1; c >= 0; c--)
|
|
zfs_free(rm->rm_col[c].rc_data, rm->rm_col[c].rc_size);
|
|
|
|
zfs_free(rm, offsetof(raidz_map_t, rm_col[rm->rm_scols]));
|
|
}
|
|
|
|
static vdev_t *
|
|
vdev_child(vdev_t *pvd, uint64_t devidx)
|
|
{
|
|
vdev_t *cvd;
|
|
|
|
STAILQ_FOREACH(cvd, &pvd->v_children, v_childlink) {
|
|
if (cvd->v_id == devidx)
|
|
break;
|
|
}
|
|
|
|
return (cvd);
|
|
}
|
|
|
|
/*
|
|
* We keep track of whether or not there were any injected errors, so that
|
|
* any ereports we generate can note it.
|
|
*/
|
|
static int
|
|
raidz_checksum_verify(const blkptr_t *bp, void *data, uint64_t size)
|
|
{
|
|
|
|
return (zio_checksum_verify(bp, data));
|
|
}
|
|
|
|
/*
|
|
* Generate the parity from the data columns. If we tried and were able to
|
|
* read the parity without error, verify that the generated parity matches the
|
|
* data we read. If it doesn't, we fire off a checksum error. Return the
|
|
* number such failures.
|
|
*/
|
|
static int
|
|
raidz_parity_verify(raidz_map_t *rm)
|
|
{
|
|
void *orig[VDEV_RAIDZ_MAXPARITY];
|
|
int c, ret = 0;
|
|
raidz_col_t *rc;
|
|
|
|
for (c = 0; c < rm->rm_firstdatacol; c++) {
|
|
rc = &rm->rm_col[c];
|
|
if (!rc->rc_tried || rc->rc_error != 0)
|
|
continue;
|
|
orig[c] = zfs_alloc(rc->rc_size);
|
|
bcopy(rc->rc_data, orig[c], rc->rc_size);
|
|
}
|
|
|
|
vdev_raidz_generate_parity(rm);
|
|
|
|
for (c = rm->rm_firstdatacol - 1; c >= 0; c--) {
|
|
rc = &rm->rm_col[c];
|
|
if (!rc->rc_tried || rc->rc_error != 0)
|
|
continue;
|
|
if (bcmp(orig[c], rc->rc_data, rc->rc_size) != 0) {
|
|
rc->rc_error = ECKSUM;
|
|
ret++;
|
|
}
|
|
zfs_free(orig[c], rc->rc_size);
|
|
}
|
|
|
|
return (ret);
|
|
}
|
|
|
|
/*
|
|
* Iterate over all combinations of bad data and attempt a reconstruction.
|
|
* Note that the algorithm below is non-optimal because it doesn't take into
|
|
* account how reconstruction is actually performed. For example, with
|
|
* triple-parity RAID-Z the reconstruction procedure is the same if column 4
|
|
* is targeted as invalid as if columns 1 and 4 are targeted since in both
|
|
* cases we'd only use parity information in column 0.
|
|
*/
|
|
static int
|
|
vdev_raidz_combrec(raidz_map_t *rm, const blkptr_t *bp, void *data,
|
|
off_t offset, uint64_t bytes, int total_errors, int data_errors)
|
|
{
|
|
raidz_col_t *rc;
|
|
void *orig[VDEV_RAIDZ_MAXPARITY];
|
|
int tstore[VDEV_RAIDZ_MAXPARITY + 2];
|
|
int *tgts = &tstore[1];
|
|
int current, next, i, c, n;
|
|
int code, ret = 0;
|
|
|
|
ASSERT(total_errors < rm->rm_firstdatacol);
|
|
|
|
/*
|
|
* This simplifies one edge condition.
|
|
*/
|
|
tgts[-1] = -1;
|
|
|
|
for (n = 1; n <= rm->rm_firstdatacol - total_errors; n++) {
|
|
/*
|
|
* Initialize the targets array by finding the first n columns
|
|
* that contain no error.
|
|
*
|
|
* If there were no data errors, we need to ensure that we're
|
|
* always explicitly attempting to reconstruct at least one
|
|
* data column. To do this, we simply push the highest target
|
|
* up into the data columns.
|
|
*/
|
|
for (c = 0, i = 0; i < n; i++) {
|
|
if (i == n - 1 && data_errors == 0 &&
|
|
c < rm->rm_firstdatacol) {
|
|
c = rm->rm_firstdatacol;
|
|
}
|
|
|
|
while (rm->rm_col[c].rc_error != 0) {
|
|
c++;
|
|
ASSERT3S(c, <, rm->rm_cols);
|
|
}
|
|
|
|
tgts[i] = c++;
|
|
}
|
|
|
|
/*
|
|
* Setting tgts[n] simplifies the other edge condition.
|
|
*/
|
|
tgts[n] = rm->rm_cols;
|
|
|
|
/*
|
|
* These buffers were allocated in previous iterations.
|
|
*/
|
|
for (i = 0; i < n - 1; i++) {
|
|
ASSERT(orig[i] != NULL);
|
|
}
|
|
|
|
orig[n - 1] = zfs_alloc(rm->rm_col[0].rc_size);
|
|
|
|
current = 0;
|
|
next = tgts[current];
|
|
|
|
while (current != n) {
|
|
tgts[current] = next;
|
|
current = 0;
|
|
|
|
/*
|
|
* Save off the original data that we're going to
|
|
* attempt to reconstruct.
|
|
*/
|
|
for (i = 0; i < n; i++) {
|
|
ASSERT(orig[i] != NULL);
|
|
c = tgts[i];
|
|
ASSERT3S(c, >=, 0);
|
|
ASSERT3S(c, <, rm->rm_cols);
|
|
rc = &rm->rm_col[c];
|
|
bcopy(rc->rc_data, orig[i], rc->rc_size);
|
|
}
|
|
|
|
/*
|
|
* Attempt a reconstruction and exit the outer loop on
|
|
* success.
|
|
*/
|
|
code = vdev_raidz_reconstruct(rm, tgts, n);
|
|
if (raidz_checksum_verify(bp, data, bytes) == 0) {
|
|
for (i = 0; i < n; i++) {
|
|
c = tgts[i];
|
|
rc = &rm->rm_col[c];
|
|
ASSERT(rc->rc_error == 0);
|
|
rc->rc_error = ECKSUM;
|
|
}
|
|
|
|
ret = code;
|
|
goto done;
|
|
}
|
|
|
|
/*
|
|
* Restore the original data.
|
|
*/
|
|
for (i = 0; i < n; i++) {
|
|
c = tgts[i];
|
|
rc = &rm->rm_col[c];
|
|
bcopy(orig[i], rc->rc_data, rc->rc_size);
|
|
}
|
|
|
|
do {
|
|
/*
|
|
* Find the next valid column after the current
|
|
* position..
|
|
*/
|
|
for (next = tgts[current] + 1;
|
|
next < rm->rm_cols &&
|
|
rm->rm_col[next].rc_error != 0; next++)
|
|
continue;
|
|
|
|
ASSERT(next <= tgts[current + 1]);
|
|
|
|
/*
|
|
* If that spot is available, we're done here.
|
|
*/
|
|
if (next != tgts[current + 1])
|
|
break;
|
|
|
|
/*
|
|
* Otherwise, find the next valid column after
|
|
* the previous position.
|
|
*/
|
|
for (c = tgts[current - 1] + 1;
|
|
rm->rm_col[c].rc_error != 0; c++)
|
|
continue;
|
|
|
|
tgts[current] = c;
|
|
current++;
|
|
|
|
} while (current != n);
|
|
}
|
|
}
|
|
n--;
|
|
done:
|
|
for (i = n - 1; i >= 0; i--) {
|
|
zfs_free(orig[i], rm->rm_col[0].rc_size);
|
|
}
|
|
|
|
return (ret);
|
|
}
|
|
|
|
static int
|
|
vdev_raidz_read(vdev_t *vd, const blkptr_t *bp, void *data,
|
|
off_t offset, size_t bytes)
|
|
{
|
|
vdev_t *tvd = vd->v_top;
|
|
vdev_t *cvd;
|
|
raidz_map_t *rm;
|
|
raidz_col_t *rc;
|
|
int c, error;
|
|
int unexpected_errors;
|
|
int parity_errors;
|
|
int parity_untried;
|
|
int data_errors;
|
|
int total_errors;
|
|
int n;
|
|
int tgts[VDEV_RAIDZ_MAXPARITY];
|
|
int code;
|
|
|
|
rc = NULL; /* gcc */
|
|
error = 0;
|
|
|
|
rm = vdev_raidz_map_alloc(data, offset, bytes, tvd->v_ashift,
|
|
vd->v_nchildren, vd->v_nparity);
|
|
|
|
/*
|
|
* Iterate over the columns in reverse order so that we hit the parity
|
|
* last -- any errors along the way will force us to read the parity.
|
|
*/
|
|
for (c = rm->rm_cols - 1; c >= 0; c--) {
|
|
rc = &rm->rm_col[c];
|
|
cvd = vdev_child(vd, rc->rc_devidx);
|
|
if (cvd == NULL || cvd->v_state != VDEV_STATE_HEALTHY) {
|
|
if (c >= rm->rm_firstdatacol)
|
|
rm->rm_missingdata++;
|
|
else
|
|
rm->rm_missingparity++;
|
|
rc->rc_error = ENXIO;
|
|
rc->rc_tried = 1; /* don't even try */
|
|
rc->rc_skipped = 1;
|
|
continue;
|
|
}
|
|
#if 0 /* XXX: Too hard for the boot code. */
|
|
if (vdev_dtl_contains(cvd, DTL_MISSING, zio->io_txg, 1)) {
|
|
if (c >= rm->rm_firstdatacol)
|
|
rm->rm_missingdata++;
|
|
else
|
|
rm->rm_missingparity++;
|
|
rc->rc_error = ESTALE;
|
|
rc->rc_skipped = 1;
|
|
continue;
|
|
}
|
|
#endif
|
|
if (c >= rm->rm_firstdatacol || rm->rm_missingdata > 0) {
|
|
rc->rc_error = cvd->v_read(cvd, NULL, rc->rc_data,
|
|
rc->rc_offset, rc->rc_size);
|
|
rc->rc_tried = 1;
|
|
rc->rc_skipped = 0;
|
|
}
|
|
}
|
|
|
|
reconstruct:
|
|
unexpected_errors = 0;
|
|
parity_errors = 0;
|
|
parity_untried = 0;
|
|
data_errors = 0;
|
|
total_errors = 0;
|
|
|
|
ASSERT(rm->rm_missingparity <= rm->rm_firstdatacol);
|
|
ASSERT(rm->rm_missingdata <= rm->rm_cols - rm->rm_firstdatacol);
|
|
|
|
for (c = 0; c < rm->rm_cols; c++) {
|
|
rc = &rm->rm_col[c];
|
|
|
|
if (rc->rc_error) {
|
|
ASSERT(rc->rc_error != ECKSUM); /* child has no bp */
|
|
|
|
if (c < rm->rm_firstdatacol)
|
|
parity_errors++;
|
|
else
|
|
data_errors++;
|
|
|
|
if (!rc->rc_skipped)
|
|
unexpected_errors++;
|
|
|
|
total_errors++;
|
|
} else if (c < rm->rm_firstdatacol && !rc->rc_tried) {
|
|
parity_untried++;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* There are three potential phases for a read:
|
|
* 1. produce valid data from the columns read
|
|
* 2. read all disks and try again
|
|
* 3. perform combinatorial reconstruction
|
|
*
|
|
* Each phase is progressively both more expensive and less likely to
|
|
* occur. If we encounter more errors than we can repair or all phases
|
|
* fail, we have no choice but to return an error.
|
|
*/
|
|
|
|
/*
|
|
* If the number of errors we saw was correctable -- less than or equal
|
|
* to the number of parity disks read -- attempt to produce data that
|
|
* has a valid checksum. Naturally, this case applies in the absence of
|
|
* any errors.
|
|
*/
|
|
if (total_errors <= rm->rm_firstdatacol - parity_untried) {
|
|
if (data_errors == 0) {
|
|
if (raidz_checksum_verify(bp, data, bytes) == 0) {
|
|
/*
|
|
* If we read parity information (unnecessarily
|
|
* as it happens since no reconstruction was
|
|
* needed) regenerate and verify the parity.
|
|
* We also regenerate parity when resilvering
|
|
* so we can write it out to the failed device
|
|
* later.
|
|
*/
|
|
if (parity_errors + parity_untried <
|
|
rm->rm_firstdatacol) {
|
|
n = raidz_parity_verify(rm);
|
|
unexpected_errors += n;
|
|
ASSERT(parity_errors + n <=
|
|
rm->rm_firstdatacol);
|
|
}
|
|
goto done;
|
|
}
|
|
} else {
|
|
/*
|
|
* We either attempt to read all the parity columns or
|
|
* none of them. If we didn't try to read parity, we
|
|
* wouldn't be here in the correctable case. There must
|
|
* also have been fewer parity errors than parity
|
|
* columns or, again, we wouldn't be in this code path.
|
|
*/
|
|
ASSERT(parity_untried == 0);
|
|
ASSERT(parity_errors < rm->rm_firstdatacol);
|
|
|
|
/*
|
|
* Identify the data columns that reported an error.
|
|
*/
|
|
n = 0;
|
|
for (c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
|
|
rc = &rm->rm_col[c];
|
|
if (rc->rc_error != 0) {
|
|
ASSERT(n < VDEV_RAIDZ_MAXPARITY);
|
|
tgts[n++] = c;
|
|
}
|
|
}
|
|
|
|
ASSERT(rm->rm_firstdatacol >= n);
|
|
|
|
code = vdev_raidz_reconstruct(rm, tgts, n);
|
|
|
|
if (raidz_checksum_verify(bp, data, bytes) == 0) {
|
|
/*
|
|
* If we read more parity disks than were used
|
|
* for reconstruction, confirm that the other
|
|
* parity disks produced correct data. This
|
|
* routine is suboptimal in that it regenerates
|
|
* the parity that we already used in addition
|
|
* to the parity that we're attempting to
|
|
* verify, but this should be a relatively
|
|
* uncommon case, and can be optimized if it
|
|
* becomes a problem. Note that we regenerate
|
|
* parity when resilvering so we can write it
|
|
* out to failed devices later.
|
|
*/
|
|
if (parity_errors < rm->rm_firstdatacol - n) {
|
|
n = raidz_parity_verify(rm);
|
|
unexpected_errors += n;
|
|
ASSERT(parity_errors + n <=
|
|
rm->rm_firstdatacol);
|
|
}
|
|
|
|
goto done;
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* This isn't a typical situation -- either we got a read
|
|
* error or a child silently returned bad data. Read every
|
|
* block so we can try again with as much data and parity as
|
|
* we can track down. If we've already been through once
|
|
* before, all children will be marked as tried so we'll
|
|
* proceed to combinatorial reconstruction.
|
|
*/
|
|
unexpected_errors = 1;
|
|
rm->rm_missingdata = 0;
|
|
rm->rm_missingparity = 0;
|
|
|
|
n = 0;
|
|
for (c = 0; c < rm->rm_cols; c++) {
|
|
rc = &rm->rm_col[c];
|
|
|
|
if (rc->rc_tried)
|
|
continue;
|
|
|
|
cvd = vdev_child(vd, rc->rc_devidx);
|
|
ASSERT(cvd != NULL);
|
|
rc->rc_error = cvd->v_read(cvd, NULL,
|
|
rc->rc_data, rc->rc_offset, rc->rc_size);
|
|
if (rc->rc_error == 0)
|
|
n++;
|
|
rc->rc_tried = 1;
|
|
rc->rc_skipped = 0;
|
|
}
|
|
/*
|
|
* If we managed to read anything more, retry the
|
|
* reconstruction.
|
|
*/
|
|
if (n > 0)
|
|
goto reconstruct;
|
|
|
|
/*
|
|
* At this point we've attempted to reconstruct the data given the
|
|
* errors we detected, and we've attempted to read all columns. There
|
|
* must, therefore, be one or more additional problems -- silent errors
|
|
* resulting in invalid data rather than explicit I/O errors resulting
|
|
* in absent data. We check if there is enough additional data to
|
|
* possibly reconstruct the data and then perform combinatorial
|
|
* reconstruction over all possible combinations. If that fails,
|
|
* we're cooked.
|
|
*/
|
|
if (total_errors > rm->rm_firstdatacol) {
|
|
error = EIO;
|
|
} else if (total_errors < rm->rm_firstdatacol &&
|
|
(code = vdev_raidz_combrec(rm, bp, data, offset, bytes,
|
|
total_errors, data_errors)) != 0) {
|
|
/*
|
|
* If we didn't use all the available parity for the
|
|
* combinatorial reconstruction, verify that the remaining
|
|
* parity is correct.
|
|
*/
|
|
if (code != (1 << rm->rm_firstdatacol) - 1)
|
|
(void) raidz_parity_verify(rm);
|
|
} else {
|
|
/*
|
|
* We're here because either:
|
|
*
|
|
* total_errors == rm_first_datacol, or
|
|
* vdev_raidz_combrec() failed
|
|
*
|
|
* In either case, there is enough bad data to prevent
|
|
* reconstruction.
|
|
*
|
|
* Start checksum ereports for all children which haven't
|
|
* failed, and the IO wasn't speculative.
|
|
*/
|
|
error = ECKSUM;
|
|
}
|
|
|
|
done:
|
|
vdev_raidz_map_free(rm);
|
|
|
|
return (error);
|
|
}
|