d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework of fletcher_4 module

Browse Source 
- Benchmark memory block is increased to 128kiB to reflect real block sizes more
accurately. Measurements include all three stages needed for checksum generation,
i.e. `init()/compute()/fini()`. The inner loop is repeated multiple times to offset
overhead of time function.

- Fastest implementation selects native and byteswap methods independently in
benchmark. To support this new function pointers `init_byteswap()/fini_byteswap()`
are introduced.

- Implementation mutex lock is replaced by atomic variable.

- To save time, benchmark is not executed in userspace. Instead, highest supported
implementation is used for fastest. Default userspace selector is still 'cycle'.

- `fletcher_4_native/byteswap()` methods use incremental methods to finish
calculation if data size is not multiple of vector stride (currently 64B).

- Added `fletcher_4_native_varsize()` special purpose method for use when buffer size
is not known in advance. The method does not enforce 4B alignment on buffer size, and
will ignore last (size % 4) bytes of the data buffer.

- Benchmark `kstat` is changed to match the one of vdev_raidz. It now shows
throughput for all supported implementations (in B/s), native and byteswap,
as well as the code [fastest] is running.

Example of `fletcher_4_bench` running on `Intel(R) Xeon(R) CPU E5-2660 v3 @ 2.60GHz`:
implementation   native         byteswap
scalar           4768120823     3426105750
sse2             7947841777     4318964249
ssse3            7951922722     6112191941
avx2             13269714358    11043200912
fastest          avx2           avx2

Example of `fletcher_4_bench` running on `Intel(R) Xeon Phi(TM) CPU 7210 @ 1.30GHz`:
implementation   native         byteswap
scalar           1291115967     1031555336
sse2             2539571138     1280970926
ssse3            2537778746     1080016762
avx2             4950749767     1078493449
avx512f          9581379998     4010029046
fastest          avx512f        avx512f

Signed-off-by: Gvozden Neskovic <neskovic@gmail.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #4952
This commit is contained in:
Gvozden Neskovic 2016-07-12 17:50:54 +02:00 committed by Brian Behlendorf
parent 70b258fc96
commit fc897b24b2
7 changed files with 385 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
include/zfs_fletcher.h
View File

@ -35,11 +35,20 @@ extern "C" {
/*
* fletcher checksum functions
*
* Note: Fletcher checksum methods expect buffer size to be 4B aligned. This
* limitation stems from the algorithm design. Performing incremental checksum
* without said alignment would yield different results. Therefore, the code
* includes assertions for the size alignment.
* For compatibility, it is required that some code paths calculate checksum of
* non-aligned buffer sizes. For this purpose, `fletcher_4_native_varsize()`
* checksum method is added. This method will ignore last (size % 4) bytes of
* the data buffer.
*/
void fletcher_2_native(const void *, uint64_t, zio_cksum_t *);
void fletcher_2_byteswap(const void *, uint64_t, zio_cksum_t *);
void fletcher_4_native(const void *, uint64_t, zio_cksum_t *);
void fletcher_4_native_varsize(const void *, uint64_t, zio_cksum_t *);
void fletcher_4_byteswap(const void *, uint64_t, zio_cksum_t *);
void fletcher_4_incremental_native(const void *, uint64_t,
zio_cksum_t *);
@ -49,14 +58,21 @@ int fletcher_4_impl_set(const char *selector);
void fletcher_4_init(void);
void fletcher_4_fini(void);
/*
* fletcher checksum struct
*/
typedef void (*fletcher_4_init_f)(zio_cksum_t *);
typedef void (*fletcher_4_fini_f)(zio_cksum_t *);
typedef void (*fletcher_4_compute_f)(const void *, uint64_t, zio_cksum_t *);
typedef struct fletcher_4_func {
void (*init)(zio_cksum_t *);
void (*fini)(zio_cksum_t *);
void (*compute)(const void *, uint64_t, zio_cksum_t *);
void (*compute_byteswap)(const void *, uint64_t, zio_cksum_t *);
fletcher_4_init_f init_native;
fletcher_4_fini_f fini_native;
fletcher_4_compute_f compute_native;
fletcher_4_init_f init_byteswap;
fletcher_4_fini_f fini_byteswap;
fletcher_4_compute_f compute_byteswap;
boolean_t (*valid)(void);
const char *name;
} fletcher_4_ops_t;

2
lib/libzfs/libzfs_sendrecv.c
View File

@ -1442,7 +1442,7 @@ zfs_send_resume_token_to_nvlist(libzfs_handle_t *hdl, const char *token)
/* verify checksum */
zio_cksum_t cksum;
fletcher_4_native(compressed, len, &cksum);
fletcher_4_native_varsize(compressed, len, &cksum);
if (cksum.zc_word[0] != checksum) {
free(compressed);
zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,

508
module/zcommon/zfs_fletcher.c
View File

@ -21,6 +21,7 @@
/*
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
* Copyright (C) 2016 Gvozden Nešković. All rights reserved.
*/
/*
@ -129,25 +130,33 @@
#include <sys/sysmacros.h>
#include <sys/byteorder.h>
#include <sys/spa.h>
#include <sys/zio_checksum.h>
#include <sys/zfs_context.h>
#include <zfs_fletcher.h>
static void fletcher_4_scalar_init(zio_cksum_t *zcp);
static void fletcher_4_scalar(const void *buf, uint64_t size,
static void fletcher_4_scalar_native(const void *buf, uint64_t size,
zio_cksum_t *zcp);
static void fletcher_4_scalar_byteswap(const void *buf, uint64_t size,
zio_cksum_t *zcp);
static boolean_t fletcher_4_scalar_valid(void);
static const fletcher_4_ops_t fletcher_4_scalar_ops = {
.init = fletcher_4_scalar_init,
.compute = fletcher_4_scalar,
.init_native = fletcher_4_scalar_init,
.compute_native = fletcher_4_scalar_native,
.init_byteswap = fletcher_4_scalar_init,
.compute_byteswap = fletcher_4_scalar_byteswap,
.valid = fletcher_4_scalar_valid,
.name = "scalar"
};
static const fletcher_4_ops_t *fletcher_4_algos[] = {
static fletcher_4_ops_t fletcher_4_fastest_impl = {
.name = "fastest",
.valid = fletcher_4_scalar_valid
};
static const fletcher_4_ops_t *fletcher_4_impls[] = {
&fletcher_4_scalar_ops,
#if defined(HAVE_SSE2)
&fletcher_4_sse2_ops,
@ -163,52 +172,39 @@ static const fletcher_4_ops_t *fletcher_4_algos[] = {
#endif
};
static enum fletcher_selector {
FLETCHER_FASTEST = 0,
FLETCHER_SCALAR,
#if defined(HAVE_SSE2)
FLETCHER_SSE2,
#endif
#if defined(HAVE_SSE2) && defined(HAVE_SSSE3)
FLETCHER_SSSE3,
#endif
#if defined(HAVE_AVX) && defined(HAVE_AVX2)
FLETCHER_AVX2,
#endif
#if defined(__x86_64) && defined(HAVE_AVX512F)
FLETCHER_AVX512F,
#endif
FLETCHER_CYCLE
} fletcher_4_impl_chosen = FLETCHER_SCALAR;
/* Hold all supported implementations */
static uint32_t fletcher_4_supp_impls_cnt = 0;
static fletcher_4_ops_t *fletcher_4_supp_impls[ARRAY_SIZE(fletcher_4_impls)];
/* Select fletcher4 implementation */
#define IMPL_FASTEST (UINT32_MAX)
#define IMPL_CYCLE (UINT32_MAX - 1)
#define IMPL_SCALAR (0)
static uint32_t fletcher_4_impl_chosen = IMPL_FASTEST;
#define IMPL_READ(i) (*(volatile uint32_t *) &(i))
static struct fletcher_4_impl_selector {
const char *fis_name;
const fletcher_4_ops_t *fis_ops;
const char *fis_name;
uint32_t fis_sel;
} fletcher_4_impl_selectors[] = {
[ FLETCHER_FASTEST ] = { "fastest", NULL },
[ FLETCHER_SCALAR ] = { "scalar", &fletcher_4_scalar_ops },
#if defined(HAVE_SSE2)
[ FLETCHER_SSE2 ] = { "sse2", &fletcher_4_sse2_ops },
#endif
#if defined(HAVE_SSE2) && defined(HAVE_SSSE3)
[ FLETCHER_SSSE3 ] = { "ssse3", &fletcher_4_ssse3_ops },
#endif
#if defined(HAVE_AVX) && defined(HAVE_AVX2)
[ FLETCHER_AVX2 ] = { "avx2", &fletcher_4_avx2_ops },
#endif
#if defined(__x86_64) && defined(HAVE_AVX512F)
[ FLETCHER_AVX512F ] = { "avx512f", &fletcher_4_avx512f_ops },
#endif
#if !defined(_KERNEL)
[ FLETCHER_CYCLE ] = { "cycle", &fletcher_4_scalar_ops }
{ "cycle", IMPL_CYCLE },
#endif
{ "fastest", IMPL_FASTEST },
{ "scalar", IMPL_SCALAR }
};
static kmutex_t fletcher_4_impl_lock;
static kstat_t *fletcher_4_kstat;
static kstat_named_t fletcher_4_kstat_data[ARRAY_SIZE(fletcher_4_algos)];
static struct fletcher_4_kstat {
uint64_t native;
uint64_t byteswap;
} fletcher_4_stat_data[ARRAY_SIZE(fletcher_4_impls) + 1];
/* Indicate that benchmark has been completed */
static boolean_t fletcher_4_initialized = B_FALSE;
void
fletcher_2_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
@ -244,13 +240,14 @@ fletcher_2_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
}
static void fletcher_4_scalar_init(zio_cksum_t *zcp)
static void
fletcher_4_scalar_init(zio_cksum_t *zcp)
{
ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
}
static void
fletcher_4_scalar(const void *buf, uint64_t size, zio_cksum_t *zcp)
fletcher_4_scalar_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
const uint32_t *ip = buf;
const uint32_t *ipend = ip + (size / sizeof (uint32_t));
@ -302,180 +299,353 @@ fletcher_4_scalar_valid(void)
int
fletcher_4_impl_set(const char *val)
{
const fletcher_4_ops_t *ops;
enum fletcher_selector idx = FLETCHER_FASTEST;
size_t val_len;
unsigned i;
int err = -EINVAL;
uint32_t impl = IMPL_READ(fletcher_4_impl_chosen);
size_t i, val_len;
val_len = strlen(val);
while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */
val_len--;
/* check mandatory implementations */
for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) {
const char *name = fletcher_4_impl_selectors[i].fis_name;
if (val_len == strlen(name) &&
strncmp(val, name, val_len) == 0) {
idx = i;
impl = fletcher_4_impl_selectors[i].fis_sel;
err = 0;
break;
}
}
if (i >= ARRAY_SIZE(fletcher_4_impl_selectors))
return (-EINVAL);
ops = fletcher_4_impl_selectors[idx].fis_ops;
if (ops == NULL || !ops->valid())
return (-ENOTSUP);
if (err != 0 && fletcher_4_initialized) {
/* check all supported implementations */
for (i = 0; i < fletcher_4_supp_impls_cnt; i++) {
const char *name = fletcher_4_supp_impls[i]->name;
mutex_enter(&fletcher_4_impl_lock);
if (fletcher_4_impl_chosen != idx)
fletcher_4_impl_chosen = idx;
mutex_exit(&fletcher_4_impl_lock);
if (val_len == strlen(name) &&
strncmp(val, name, val_len) == 0) {
impl = i;
err = 0;
break;
}
}
}
return (0);
if (err == 0) {
atomic_swap_32(&fletcher_4_impl_chosen, impl);
membar_producer();
}
return (err);
}
static inline const fletcher_4_ops_t *
fletcher_4_impl_get(void)
{
fletcher_4_ops_t *ops = NULL;
const uint32_t impl = IMPL_READ(fletcher_4_impl_chosen);
switch (impl) {
case IMPL_FASTEST:
ASSERT(fletcher_4_initialized);
ops = &fletcher_4_fastest_impl;
break;
#if !defined(_KERNEL)
if (fletcher_4_impl_chosen == FLETCHER_CYCLE) {
static volatile unsigned int cycle_count = 0;
const fletcher_4_ops_t *ops = NULL;
unsigned int index;
case IMPL_CYCLE: {
ASSERT(fletcher_4_initialized);
ASSERT3U(fletcher_4_supp_impls_cnt, >, 0);
while (1) {
index = atomic_inc_uint_nv(&cycle_count);
ops = fletcher_4_algos[
index % ARRAY_SIZE(fletcher_4_algos)];
if (ops->valid())
break;
}
return (ops);
static uint32_t cycle_count = 0;
uint32_t idx = (++cycle_count) % fletcher_4_supp_impls_cnt;
ops = fletcher_4_supp_impls[idx];
}
break;
#endif
membar_producer();
return (fletcher_4_impl_selectors[fletcher_4_impl_chosen].fis_ops);
}
default:
ASSERT3U(fletcher_4_supp_impls_cnt, >, 0);
ASSERT3U(impl, <, fletcher_4_supp_impls_cnt);
void
fletcher_4_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
const fletcher_4_ops_t *ops;
ops = fletcher_4_supp_impls[impl];
break;
}
if (IS_P2ALIGNED(size, 8 * sizeof (uint32_t)))
ops = fletcher_4_impl_get();
else
ops = &fletcher_4_scalar_ops;
ASSERT3P(ops, !=, NULL);
ops->init(zcp);
ops->compute(buf, size, zcp);
if (ops->fini != NULL)
ops->fini(zcp);
}
void
fletcher_4_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
const fletcher_4_ops_t *ops;
if (IS_P2ALIGNED(size, 8 * sizeof (uint32_t)))
ops = fletcher_4_impl_get();
else
ops = &fletcher_4_scalar_ops;
ops->init(zcp);
ops->compute_byteswap(buf, size, zcp);
if (ops->fini != NULL)
ops->fini(zcp);
return (ops);
}
void
fletcher_4_incremental_native(const void *buf, uint64_t size,
zio_cksum_t *zcp)
{
fletcher_4_scalar(buf, size, zcp);
ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t)));
fletcher_4_scalar_native(buf, size, zcp);
}
void
fletcher_4_incremental_byteswap(const void *buf, uint64_t size,
zio_cksum_t *zcp)
{
ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t)));
fletcher_4_scalar_byteswap(buf, size, zcp);
}
static inline void
fletcher_4_native_impl(const fletcher_4_ops_t *ops, const void *buf,
uint64_t size, zio_cksum_t *zcp)
{
ops->init_native(zcp);
ops->compute_native(buf, size, zcp);
if (ops->fini_native != NULL)
ops->fini_native(zcp);
}
void
fletcher_4_native(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
const fletcher_4_ops_t *ops;
uint64_t p2size = P2ALIGN(size, 64);
ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t)));
if (size == 0) {
ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
} else if (p2size == 0) {
ops = &fletcher_4_scalar_ops;
fletcher_4_native_impl(ops, buf, size, zcp);
} else {
ops = fletcher_4_impl_get();
fletcher_4_native_impl(ops, buf, p2size, zcp);
if (p2size < size)
fletcher_4_incremental_native((char *)buf + p2size,
size - p2size, zcp);
}
}
void
fletcher_4_native_varsize(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
fletcher_4_native_impl(&fletcher_4_scalar_ops, buf, size, zcp);
}
static inline void
fletcher_4_byteswap_impl(const fletcher_4_ops_t *ops, const void *buf,
uint64_t size, zio_cksum_t *zcp)
{
ops->init_byteswap(zcp);
ops->compute_byteswap(buf, size, zcp);
if (ops->fini_byteswap != NULL)
ops->fini_byteswap(zcp);
}
void
fletcher_4_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp)
{
const fletcher_4_ops_t *ops;
uint64_t p2size = P2ALIGN(size, 64);
ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t)));
if (size == 0) {
ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
} else if (p2size == 0) {
ops = &fletcher_4_scalar_ops;
fletcher_4_byteswap_impl(ops, buf, size, zcp);
} else {
ops = fletcher_4_impl_get();
fletcher_4_byteswap_impl(ops, buf, p2size, zcp);
if (p2size < size)
fletcher_4_incremental_byteswap((char *)buf + p2size,
size - p2size, zcp);
}
}
static int
fletcher_4_kstat_headers(char *buf, size_t size)
{
ssize_t off = 0;
off += snprintf(buf + off, size, "%-17s", "implementation");
off += snprintf(buf + off, size - off, "%-15s", "native");
(void) snprintf(buf + off, size - off, "%-15s\n", "byteswap");
return (0);
}
static int
fletcher_4_kstat_data(char *buf, size_t size, void *data)
{
struct fletcher_4_kstat *fastest_stat =
&fletcher_4_stat_data[fletcher_4_supp_impls_cnt];
struct fletcher_4_kstat *curr_stat = (struct fletcher_4_kstat *) data;
ssize_t off = 0;
if (curr_stat == fastest_stat) {
off += snprintf(buf + off, size - off, "%-17s", "fastest");
off += snprintf(buf + off, size - off, "%-15s",
fletcher_4_supp_impls[fastest_stat->native]->name);
off += snprintf(buf + off, size - off, "%-15s\n",
fletcher_4_supp_impls[fastest_stat->byteswap]->name);
} else {
ptrdiff_t id = curr_stat - fletcher_4_stat_data;
off += snprintf(buf + off, size - off, "%-17s",
fletcher_4_supp_impls[id]->name);
off += snprintf(buf + off, size - off, "%-15llu",
(u_longlong_t) curr_stat->native);
off += snprintf(buf + off, size - off, "%-15llu\n",
(u_longlong_t) curr_stat->byteswap);
}
return (0);
}
static void *
fletcher_4_kstat_addr(kstat_t *ksp, loff_t n)
{
if (n <= fletcher_4_supp_impls_cnt)
ksp->ks_private = (void *) (fletcher_4_stat_data + n);
else
ksp->ks_private = NULL;
return (ksp->ks_private);
}
#define FLETCHER_4_FASTEST_FN_COPY(type, src) \
{ \
fletcher_4_fastest_impl.init_ ## type = src->init_ ## type; \
fletcher_4_fastest_impl.fini_ ## type = src->fini_ ## type; \
fletcher_4_fastest_impl.compute_ ## type = src->compute_ ## type; \
}
#define FLETCHER_4_BENCH_NS (MSEC2NSEC(50)) /* 50ms */
static void
fletcher_4_benchmark_impl(boolean_t native, char *data, uint64_t data_size)
{
struct fletcher_4_kstat *fastest_stat =
&fletcher_4_stat_data[fletcher_4_supp_impls_cnt];
hrtime_t start;
uint64_t run_bw, run_time_ns, best_run = 0;
zio_cksum_t zc;
uint32_t i, l, sel_save = IMPL_READ(fletcher_4_impl_chosen);
zio_checksum_func_t *fletcher_4_test = native ? fletcher_4_native :
fletcher_4_byteswap;
for (i = 0; i < fletcher_4_supp_impls_cnt; i++) {
struct fletcher_4_kstat *stat = &fletcher_4_stat_data[i];
uint64_t run_count = 0;
/* temporary set an implementation */
fletcher_4_impl_chosen = i;
kpreempt_disable();
start = gethrtime();
do {
for (l = 0; l < 32; l++, run_count++)
fletcher_4_test(data, data_size, &zc);
run_time_ns = gethrtime() - start;
} while (run_time_ns < FLETCHER_4_BENCH_NS);
kpreempt_enable();
run_bw = data_size * run_count * NANOSEC;
run_bw /= run_time_ns; /* B/s */
if (native)
stat->native = run_bw;
else
stat->byteswap = run_bw;
if (run_bw > best_run) {
best_run = run_bw;
if (native) {
fastest_stat->native = i;
FLETCHER_4_FASTEST_FN_COPY(native,
fletcher_4_supp_impls[i]);
} else {
fastest_stat->byteswap = i;
FLETCHER_4_FASTEST_FN_COPY(byteswap,
fletcher_4_supp_impls[i]);
}
}
}
/* restore original selection */
atomic_swap_32(&fletcher_4_impl_chosen, sel_save);
}
void
fletcher_4_init(void)
{
const uint64_t const bench_ns = (50 * MICROSEC); /* 50ms */
unsigned long best_run_count = 0;
unsigned long best_run_index = 0;
const unsigned data_size = 4096;
static const size_t data_size = 1 << SPA_OLD_MAXBLOCKSHIFT; /* 128kiB */
fletcher_4_ops_t *curr_impl;
char *databuf;
int i;
int i, c;
databuf = kmem_alloc(data_size, KM_SLEEP);
for (i = 0; i < ARRAY_SIZE(fletcher_4_algos); i++) {
const fletcher_4_ops_t *ops = fletcher_4_algos[i];
kstat_named_t *stat = &fletcher_4_kstat_data[i];
unsigned long run_count = 0;
hrtime_t start;
zio_cksum_t zc;
/* move supported impl into fletcher_4_supp_impls */
for (i = 0, c = 0; i < ARRAY_SIZE(fletcher_4_impls); i++) {
curr_impl = (fletcher_4_ops_t *) fletcher_4_impls[i];
strncpy(stat->name, ops->name, sizeof (stat->name) - 1);
stat->data_type = KSTAT_DATA_UINT64;
stat->value.ui64 = 0;
if (!ops->valid())
continue;
kpreempt_disable();
start = gethrtime();
ops->init(&zc);
do {
ops->compute(databuf, data_size, &zc);
ops->compute_byteswap(databuf, data_size, &zc);
run_count++;
} while (gethrtime() < start + bench_ns);
if (ops->fini != NULL)
ops->fini(&zc);
kpreempt_enable();
if (run_count > best_run_count) {
best_run_count = run_count;
best_run_index = i;
}
/*
* Due to high overhead of gethrtime(), the performance data
* here is inaccurate and much slower than it could be.
* It's fine for our use though because only relative speed
* is important.
*/
stat->value.ui64 = data_size * run_count *
(NANOSEC / bench_ns) >> 20; /* by MB/s */
if (curr_impl->valid && curr_impl->valid())
fletcher_4_supp_impls[c++] = curr_impl;
}
kmem_free(databuf, data_size);
membar_producer(); /* complete fletcher_4_supp_impls[] init */
fletcher_4_supp_impls_cnt = c; /* number of supported impl */
fletcher_4_impl_selectors[FLETCHER_FASTEST].fis_ops =
fletcher_4_algos[best_run_index];
#if !defined(_KERNEL)
/* Skip benchmarking and use last implementation as fastest */
memcpy(&fletcher_4_fastest_impl,
fletcher_4_supp_impls[fletcher_4_supp_impls_cnt-1],
sizeof (fletcher_4_fastest_impl));
fletcher_4_fastest_impl.name = "fastest";
membar_producer();
mutex_init(&fletcher_4_impl_lock, NULL, MUTEX_DEFAULT, NULL);
fletcher_4_impl_set("fastest");
fletcher_4_initialized = B_TRUE;
fletcher_4_kstat = kstat_create("zfs", 0, "fletcher_4_bench",
"misc", KSTAT_TYPE_NAMED, ARRAY_SIZE(fletcher_4_algos),
KSTAT_FLAG_VIRTUAL);
/* Use 'cycle' math selection method for userspace */
VERIFY0(fletcher_4_impl_set("cycle"));
return;
#endif
/* Benchmark all supported implementations */
databuf = vmem_alloc(data_size, KM_SLEEP);
for (i = 0; i < data_size / sizeof (uint64_t); i++)
((uint64_t *)databuf)[i] = (uintptr_t)(databuf+i); /* warm-up */
fletcher_4_benchmark_impl(B_FALSE, databuf, data_size);
fletcher_4_benchmark_impl(B_TRUE, databuf, data_size);
vmem_free(databuf, data_size);
/* install kstats for all implementations */
fletcher_4_kstat = kstat_create("zfs", 0, "fletcher_4_bench", "misc",
KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);
if (fletcher_4_kstat != NULL) {
fletcher_4_kstat->ks_data = fletcher_4_kstat_data;
fletcher_4_kstat->ks_data = NULL;
fletcher_4_kstat->ks_ndata = UINT32_MAX;
kstat_set_raw_ops(fletcher_4_kstat,
fletcher_4_kstat_headers,
fletcher_4_kstat_data,
fletcher_4_kstat_addr);
kstat_install(fletcher_4_kstat);
}
/* Finish initialization */
fletcher_4_initialized = B_TRUE;
}
void
fletcher_4_fini(void)
{
mutex_destroy(&fletcher_4_impl_lock);
if (fletcher_4_kstat != NULL) {
kstat_delete(fletcher_4_kstat);
fletcher_4_kstat = NULL;
@ -487,18 +657,19 @@ fletcher_4_fini(void)
static int
fletcher_4_param_get(char *buffer, struct kernel_param *unused)
{
const uint32_t impl = IMPL_READ(fletcher_4_impl_chosen);
char *fmt;
int i, cnt = 0;
for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) {
const fletcher_4_ops_t *ops;
/* list fastest */
fmt = (impl == IMPL_FASTEST) ? "[%s] " : "%s ";
cnt += sprintf(buffer + cnt, fmt, "fastest");
ops = fletcher_4_impl_selectors[i].fis_ops;
if (!ops->valid())
continue;
cnt += sprintf(buffer + cnt,
fletcher_4_impl_chosen == i ? "[%s] " : "%s ",
fletcher_4_impl_selectors[i].fis_name);
/* list all supported implementations */
for (i = 0; i < fletcher_4_supp_impls_cnt; i++) {
fmt = (i == impl) ? "[%s] " : "%s ";
cnt += sprintf(buffer + cnt, fmt,
fletcher_4_supp_impls[i]->name);
}
return (cnt);
@ -512,20 +683,19 @@ fletcher_4_param_set(const char *val, struct kernel_param *unused)
/*
* Choose a fletcher 4 implementation in ZFS.
* Users can choose the "fastest" algorithm, or "scalar" and "avx2" which means
* to compute fletcher 4 by CPU or vector instructions respectively.
* Users can also choose "cycle" to exercise all implementions, but this is
* Users can choose "cycle" to exercise all implementations, but this is
* for testing purpose therefore it can only be set in user space.
*/
module_param_call(zfs_fletcher_4_impl,
fletcher_4_param_set, fletcher_4_param_get, NULL, 0644);
MODULE_PARM_DESC(zfs_fletcher_4_impl, "Select fletcher 4 algorithm");
MODULE_PARM_DESC(zfs_fletcher_4_impl, "Select fletcher 4 implementation.");
EXPORT_SYMBOL(fletcher_4_init);
EXPORT_SYMBOL(fletcher_4_fini);
EXPORT_SYMBOL(fletcher_2_native);
EXPORT_SYMBOL(fletcher_2_byteswap);
EXPORT_SYMBOL(fletcher_4_native);
EXPORT_SYMBOL(fletcher_4_native_varsize);
EXPORT_SYMBOL(fletcher_4_byteswap);
EXPORT_SYMBOL(fletcher_4_incremental_native);
EXPORT_SYMBOL(fletcher_4_incremental_byteswap);

10
module/zcommon/zfs_fletcher_avx512.c
View File

@ -48,7 +48,7 @@ fletcher_4_avx512f_init(zio_cksum_t *zcp)
}
static void
fletcher_4_avx512f(const void *buf, uint64_t size, zio_cksum_t *unused)
fletcher_4_avx512f_native(const void *buf, uint64_t size, zio_cksum_t *unused)
{
const uint32_t *ip = buf;
const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size);
@ -146,9 +146,11 @@ fletcher_4_avx512f_valid(void)
}
const fletcher_4_ops_t fletcher_4_avx512f_ops = {
.init = fletcher_4_avx512f_init,
.fini = fletcher_4_avx512f_fini,
.compute = fletcher_4_avx512f,
.init_native = fletcher_4_avx512f_init,
.fini_native = fletcher_4_avx512f_fini,
.compute_native = fletcher_4_avx512f_native,
.init_byteswap = fletcher_4_avx512f_init,
.fini_byteswap = fletcher_4_avx512f_fini,
.compute_byteswap = fletcher_4_avx512f_byteswap,
.valid = fletcher_4_avx512f_valid,
.name = "avx512f"

10
module/zcommon/zfs_fletcher_intel.c
View File

@ -92,7 +92,7 @@ fletcher_4_avx2_fini(zio_cksum_t *zcp)
}
static void
fletcher_4_avx2(const void *buf, uint64_t size, zio_cksum_t *unused)
fletcher_4_avx2_native(const void *buf, uint64_t size, zio_cksum_t *unused)
{
const uint64_t *ip = buf;
const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size);
@ -137,9 +137,11 @@ static boolean_t fletcher_4_avx2_valid(void)
}
const fletcher_4_ops_t fletcher_4_avx2_ops = {
.init = fletcher_4_avx2_init,
.fini = fletcher_4_avx2_fini,
.compute = fletcher_4_avx2,
.init_native = fletcher_4_avx2_init,
.fini_native = fletcher_4_avx2_fini,
.compute_native = fletcher_4_avx2_native,
.init_byteswap = fletcher_4_avx2_init,
.fini_byteswap = fletcher_4_avx2_fini,
.compute_byteswap = fletcher_4_avx2_byteswap,
.valid = fletcher_4_avx2_valid,
.name = "avx2"

18
module/zcommon/zfs_fletcher_sse.c
View File

@ -97,7 +97,7 @@ fletcher_4_sse2_fini(zio_cksum_t *zcp)
}
static void
fletcher_4_sse2(const void *buf, uint64_t size, zio_cksum_t *unused)
fletcher_4_sse2_native(const void *buf, uint64_t size, zio_cksum_t *unused)
{
const uint64_t *ip = buf;
const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size);
@ -147,9 +147,11 @@ static boolean_t fletcher_4_sse2_valid(void)
}
const fletcher_4_ops_t fletcher_4_sse2_ops = {
.init = fletcher_4_sse2_init,
.fini = fletcher_4_sse2_fini,
.compute = fletcher_4_sse2,
.init_native = fletcher_4_sse2_init,
.fini_native = fletcher_4_sse2_fini,
.compute_native = fletcher_4_sse2_native,
.init_byteswap = fletcher_4_sse2_init,
.fini_byteswap = fletcher_4_sse2_fini,
.compute_byteswap = fletcher_4_sse2_byteswap,
.valid = fletcher_4_sse2_valid,
.name = "sse2"
@ -194,9 +196,11 @@ static boolean_t fletcher_4_ssse3_valid(void)
}
const fletcher_4_ops_t fletcher_4_ssse3_ops = {
.init = fletcher_4_sse2_init,
.fini = fletcher_4_sse2_fini,
.compute = fletcher_4_sse2,
.init_native = fletcher_4_sse2_init,
.fini_native = fletcher_4_sse2_fini,
.compute_native = fletcher_4_sse2_native,
.init_byteswap = fletcher_4_sse2_init,
.fini_byteswap = fletcher_4_sse2_fini,
.compute_byteswap = fletcher_4_ssse3_byteswap,
.valid = fletcher_4_ssse3_valid,
.name = "ssse3"

2
module/zfs/dsl_dataset.c
View File

@ -1770,7 +1770,7 @@ get_receive_resume_stats(dsl_dataset_t *ds, nvlist_t *nv)
compressed_size = gzip_compress(packed, compressed,
packed_size, packed_size, 6);
fletcher_4_native(compressed, compressed_size, &cksum);
fletcher_4_native_varsize(compressed, compressed_size, &cksum);
str = kmem_alloc(compressed_size * 2 + 1, KM_SLEEP);
for (i = 0; i < compressed_size; i++) {