Add generic implementation handling and SHA2 impl
The skeleton file module/icp/include/generic_impl.c can be used for iterating over different implementations of algorithms. It is used by SHA256, SHA512 and BLAKE3 currently. The Solaris SHA2 implementation got replaced with a version which is based on public domain code of cppcrypto v0.10. These assembly files are taken from current openssl master: - sha256-x86_64.S: x64, SSSE3, AVX, AVX2, SHA-NI (x86_64) - sha512-x86_64.S: x64, AVX, AVX2 (x86_64) - sha256-armv7.S: ARMv7, NEON, ARMv8-CE (arm) - sha512-armv7.S: ARMv7, NEON (arm) - sha256-armv8.S: ARMv7, NEON, ARMv8-CE (aarch64) - sha512-armv8.S: ARMv7, ARMv8-CE (aarch64) - sha256-ppc.S: Generic PPC64 LE/BE (ppc64) - sha512-ppc.S: Generic PPC64 LE/BE (ppc64) - sha256-p8.S: Power8 ISA Version 2.07 LE/BE (ppc64) - sha512-p8.S: Power8 ISA Version 2.07 LE/BE (ppc64) Tested-by: Rich Ercolani <rincebrain@gmail.com> Tested-by: Sebastian Gottschall <s.gottschall@dd-wrt.com> Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov> Signed-off-by: Tino Reichardt <milky-zfs@mcmilk.de> Closes #13741
This commit is contained in:
parent
ac678c8eee
commit
4c5fec01a4
@ -75,6 +75,7 @@ COMMON_H = \
|
||||
sys/rrwlock.h \
|
||||
sys/sa.h \
|
||||
sys/sa_impl.h \
|
||||
sys/sha2.h \
|
||||
sys/skein.h \
|
||||
sys/spa.h \
|
||||
sys/spa_checkpoint.h \
|
||||
@ -124,6 +125,7 @@ COMMON_H = \
|
||||
sys/zfs_delay.h \
|
||||
sys/zfs_file.h \
|
||||
sys/zfs_fuid.h \
|
||||
sys/zfs_impl.h \
|
||||
sys/zfs_project.h \
|
||||
sys/zfs_quota.h \
|
||||
sys/zfs_racct.h \
|
||||
|
127
include/sys/sha2.h
Normal file
127
include/sys/sha2.h
Normal file
@ -0,0 +1,127 @@
|
||||
/*
|
||||
* CDDL HEADER START
|
||||
*
|
||||
* The contents of this file are subject to the terms of the
|
||||
* Common Development and Distribution License (the "License").
|
||||
* You may not use this file except in compliance with the License.
|
||||
*
|
||||
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
||||
* or https://opensource.org/licenses/CDDL-1.0.
|
||||
* See the License for the specific language governing permissions
|
||||
* and limitations under the License.
|
||||
*
|
||||
* When distributing Covered Code, include this CDDL HEADER in each
|
||||
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
||||
* If applicable, add the following below this CDDL HEADER, with the
|
||||
* fields enclosed by brackets "[]" replaced with your own identifying
|
||||
* information: Portions Copyright [yyyy] [name of copyright owner]
|
||||
*
|
||||
* CDDL HEADER END
|
||||
*/
|
||||
|
||||
/*
|
||||
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
|
||||
*/
|
||||
|
||||
#ifndef _SYS_SHA2_H
|
||||
#define _SYS_SHA2_H
|
||||
|
||||
#ifdef _KERNEL
|
||||
#include <sys/types.h>
|
||||
#else
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define SHA224_BLOCK_LENGTH 64
|
||||
#define SHA256_BLOCK_LENGTH 64
|
||||
#define SHA384_BLOCK_LENGTH 128
|
||||
#define SHA512_BLOCK_LENGTH 128
|
||||
|
||||
#define SHA224_DIGEST_LENGTH 28
|
||||
#define SHA256_DIGEST_LENGTH 32
|
||||
#define SHA384_DIGEST_LENGTH 48
|
||||
#define SHA512_DIGEST_LENGTH 64
|
||||
|
||||
#define SHA512_224_DIGEST_LENGTH 28
|
||||
#define SHA512_256_DIGEST_LENGTH 32
|
||||
|
||||
#define SHA256_HMAC_BLOCK_SIZE 64
|
||||
#define SHA512_HMAC_BLOCK_SIZE 128
|
||||
|
||||
/* sha256 context */
|
||||
typedef struct {
|
||||
uint32_t state[8];
|
||||
uint64_t count[2];
|
||||
uint8_t wbuf[64];
|
||||
|
||||
/* const sha256_ops_t *ops */
|
||||
const void *ops;
|
||||
} sha256_ctx;
|
||||
|
||||
/* sha512 context */
|
||||
typedef struct {
|
||||
uint64_t state[8];
|
||||
uint64_t count[2];
|
||||
uint8_t wbuf[128];
|
||||
|
||||
/* const sha256_ops_t *ops */
|
||||
const void *ops;
|
||||
} sha512_ctx;
|
||||
|
||||
/* SHA2 context */
|
||||
typedef struct {
|
||||
union {
|
||||
sha256_ctx sha256;
|
||||
sha512_ctx sha512;
|
||||
};
|
||||
|
||||
/* algorithm type */
|
||||
int algotype;
|
||||
} SHA2_CTX;
|
||||
|
||||
/* SHA2 algorithm types */
|
||||
typedef enum sha2_mech_type {
|
||||
SHA256_MECH_INFO_TYPE, /* SUN_CKM_SHA256 */
|
||||
SHA256_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC */
|
||||
SHA256_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC_GENERAL */
|
||||
SHA384_MECH_INFO_TYPE, /* SUN_CKM_SHA384 */
|
||||
SHA384_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA384_HMAC */
|
||||
SHA384_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA384_HMAC_GENERAL */
|
||||
SHA512_MECH_INFO_TYPE, /* SUN_CKM_SHA512 */
|
||||
SHA512_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA512_HMAC */
|
||||
SHA512_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA512_HMAC_GENERAL */
|
||||
SHA512_224_MECH_INFO_TYPE, /* SUN_CKM_SHA512_224 */
|
||||
SHA512_256_MECH_INFO_TYPE /* SUN_CKM_SHA512_256 */
|
||||
} sha2_mech_type_t;
|
||||
|
||||
#define SHA256 0
|
||||
#define SHA256_HMAC 1
|
||||
#define SHA256_HMAC_GEN 2
|
||||
#define SHA384 3
|
||||
#define SHA384_HMAC 4
|
||||
#define SHA384_HMAC_GEN 5
|
||||
#define SHA512 6
|
||||
#define SHA512_HMAC 7
|
||||
#define SHA512_HMAC_GEN 8
|
||||
#define SHA512_224 9
|
||||
#define SHA512_256 10
|
||||
|
||||
/* SHA2 Init function */
|
||||
extern void SHA2Init(int algotype, SHA2_CTX *ctx);
|
||||
|
||||
/* SHA2 Update function */
|
||||
extern void SHA2Update(SHA2_CTX *ctx, const void *data, size_t len);
|
||||
|
||||
/* SHA2 Final function */
|
||||
extern void SHA2Final(void *digest, SHA2_CTX *ctx);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* SYS_SHA2_H */
|
69
include/sys/zfs_impl.h
Normal file
69
include/sys/zfs_impl.h
Normal file
@ -0,0 +1,69 @@
|
||||
/*
|
||||
* CDDL HEADER START
|
||||
*
|
||||
* The contents of this file are subject to the terms of the
|
||||
* Common Development and Distribution License (the "License").
|
||||
* You may not use this file except in compliance with the License.
|
||||
*
|
||||
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
||||
* or https://opensource.org/licenses/CDDL-1.0.
|
||||
* See the License for the specific language governing permissions
|
||||
* and limitations under the License.
|
||||
*
|
||||
* When distributing Covered Code, include this CDDL HEADER in each
|
||||
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
||||
* If applicable, add the following below this CDDL HEADER, with the
|
||||
* fields enclosed by brackets "[]" replaced with your own identifying
|
||||
* information: Portions Copyright [yyyy] [name of copyright owner]
|
||||
*
|
||||
* CDDL HEADER END
|
||||
*/
|
||||
|
||||
/*
|
||||
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
|
||||
*/
|
||||
|
||||
#ifndef _SYS_ZFS_IMPL_H
|
||||
#define _SYS_ZFS_IMPL_H
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* generic implementation backends */
|
||||
typedef struct
|
||||
{
|
||||
/* algorithm name */
|
||||
const char *name;
|
||||
|
||||
/* get number of supported implementations */
|
||||
uint32_t (*getcnt)(void);
|
||||
|
||||
/* get id of selected implementation */
|
||||
uint32_t (*getid)(void);
|
||||
|
||||
/* get name of selected implementation */
|
||||
const char *(*getname)(void);
|
||||
|
||||
/* setup id as fastest implementation */
|
||||
void (*set_fastest)(uint32_t id);
|
||||
|
||||
/* set implementation by id */
|
||||
void (*setid)(uint32_t id);
|
||||
|
||||
/* set implementation by name */
|
||||
int (*setname)(const char *val);
|
||||
} zfs_impl_t;
|
||||
|
||||
/* return some set of function pointer */
|
||||
extern const zfs_impl_t *zfs_impl_get_ops(const char *algo);
|
||||
|
||||
extern const zfs_impl_t zfs_blake3_ops;
|
||||
extern const zfs_impl_t zfs_sha256_ops;
|
||||
extern const zfs_impl_t zfs_sha512_ops;
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* _SYS_ZFS_IMPL_H */
|
@ -110,9 +110,9 @@ _SYS_ZIO_CHECKSUM_H zio_checksum_info_t
|
||||
*/
|
||||
|
||||
/* SHA2 */
|
||||
extern zio_checksum_t abd_checksum_SHA256;
|
||||
extern zio_checksum_t abd_checksum_SHA512_native;
|
||||
extern zio_checksum_t abd_checksum_SHA512_byteswap;
|
||||
extern zio_checksum_t abd_checksum_sha256;
|
||||
extern zio_checksum_t abd_checksum_sha512_native;
|
||||
extern zio_checksum_t abd_checksum_sha512_byteswap;
|
||||
|
||||
/* Skein */
|
||||
extern zio_checksum_t abd_checksum_skein_native;
|
||||
|
@ -16,7 +16,6 @@ nodist_libicp_la_SOURCES = \
|
||||
module/icp/algs/blake3/blake3.c \
|
||||
module/icp/algs/blake3/blake3_generic.c \
|
||||
module/icp/algs/blake3/blake3_impl.c \
|
||||
module/icp/algs/blake3/blake3_x86-64.c \
|
||||
module/icp/algs/edonr/edonr.c \
|
||||
module/icp/algs/modes/modes.c \
|
||||
module/icp/algs/modes/cbc.c \
|
||||
@ -26,7 +25,9 @@ nodist_libicp_la_SOURCES = \
|
||||
module/icp/algs/modes/ctr.c \
|
||||
module/icp/algs/modes/ccm.c \
|
||||
module/icp/algs/modes/ecb.c \
|
||||
module/icp/algs/sha2/sha2.c \
|
||||
module/icp/algs/sha2/sha2_generic.c \
|
||||
module/icp/algs/sha2/sha256_impl.c \
|
||||
module/icp/algs/sha2/sha512_impl.c \
|
||||
module/icp/algs/skein/skein.c \
|
||||
module/icp/algs/skein/skein_block.c \
|
||||
module/icp/algs/skein/skein_iv.c \
|
||||
@ -38,18 +39,31 @@ nodist_libicp_la_SOURCES = \
|
||||
module/icp/core/kcf_prov_lib.c \
|
||||
module/icp/core/kcf_callprov.c \
|
||||
module/icp/core/kcf_mech_tabs.c \
|
||||
module/icp/core/kcf_prov_tabs.c
|
||||
module/icp/core/kcf_prov_tabs.c \
|
||||
module/zfs/zfs_impl.c
|
||||
|
||||
if TARGET_CPU_AARCH64
|
||||
nodist_libicp_la_SOURCES += \
|
||||
module/icp/asm-aarch64/blake3/b3_aarch64_sse2.S \
|
||||
module/icp/asm-aarch64/blake3/b3_aarch64_sse41.S
|
||||
module/icp/asm-aarch64/blake3/b3_aarch64_sse41.S \
|
||||
module/icp/asm-aarch64/sha2/sha256-armv8.S \
|
||||
module/icp/asm-aarch64/sha2/sha512-armv8.S
|
||||
endif
|
||||
|
||||
if TARGET_CPU_ARM
|
||||
nodist_libicp_la_SOURCES += \
|
||||
module/icp/asm-arm/sha2/sha256-armv7.S \
|
||||
module/icp/asm-arm/sha2/sha512-armv7.S
|
||||
endif
|
||||
|
||||
if TARGET_CPU_POWERPC
|
||||
nodist_libicp_la_SOURCES += \
|
||||
module/icp/asm-ppc64/blake3/b3_ppc64le_sse2.S \
|
||||
module/icp/asm-ppc64/blake3/b3_ppc64le_sse41.S
|
||||
module/icp/asm-ppc64/blake3/b3_ppc64le_sse41.S \
|
||||
module/icp/asm-ppc64/sha2/sha256-ppc.S \
|
||||
module/icp/asm-ppc64/sha2/sha512-ppc.S \
|
||||
module/icp/asm-ppc64/sha2/sha256-p8.S \
|
||||
module/icp/asm-ppc64/sha2/sha512-p8.S
|
||||
endif
|
||||
|
||||
if TARGET_CPU_X86_64
|
||||
@ -60,8 +74,8 @@ nodist_libicp_la_SOURCES += \
|
||||
module/icp/asm-x86_64/modes/gcm_pclmulqdq.S \
|
||||
module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S \
|
||||
module/icp/asm-x86_64/modes/ghash-x86_64.S \
|
||||
module/icp/asm-x86_64/sha2/sha256_impl.S \
|
||||
module/icp/asm-x86_64/sha2/sha512_impl.S \
|
||||
module/icp/asm-x86_64/sha2/sha256-x86_64.S \
|
||||
module/icp/asm-x86_64/sha2/sha512-x86_64.S \
|
||||
module/icp/asm-x86_64/blake3/blake3_avx2.S \
|
||||
module/icp/asm-x86_64/blake3/blake3_avx512.S \
|
||||
module/icp/asm-x86_64/blake3/blake3_sse2.S \
|
||||
|
@ -34,8 +34,6 @@ dist_libzfs_la_SOURCES += \
|
||||
endif
|
||||
|
||||
nodist_libzfs_la_SOURCES = \
|
||||
module/icp/algs/sha2/sha2.c \
|
||||
\
|
||||
module/zcommon/cityhash.c \
|
||||
module/zcommon/zfeature_common.c \
|
||||
module/zcommon/zfs_comutil.c \
|
||||
@ -52,7 +50,6 @@ nodist_libzfs_la_SOURCES = \
|
||||
module/zcommon/zpool_prop.c \
|
||||
module/zcommon/zprop_common.c
|
||||
|
||||
|
||||
libzfs_la_LIBADD = \
|
||||
libshare.la \
|
||||
libzfs_core.la \
|
||||
|
@ -118,7 +118,7 @@ nodist_libzpool_la_SOURCES = \
|
||||
module/zfs/refcount.c \
|
||||
module/zfs/rrwlock.c \
|
||||
module/zfs/sa.c \
|
||||
module/zfs/sha256.c \
|
||||
module/zfs/sha2_zfs.c \
|
||||
module/zfs/skein_zfs.c \
|
||||
module/zfs/spa.c \
|
||||
module/zfs/spa_checkpoint.c \
|
||||
|
@ -85,7 +85,6 @@ ICP_OBJS := \
|
||||
algs/blake3/blake3.o \
|
||||
algs/blake3/blake3_generic.o \
|
||||
algs/blake3/blake3_impl.o \
|
||||
algs/blake3/blake3_x86-64.o \
|
||||
algs/edonr/edonr.o \
|
||||
algs/modes/cbc.o \
|
||||
algs/modes/ccm.o \
|
||||
@ -94,6 +93,9 @@ ICP_OBJS := \
|
||||
algs/modes/gcm.o \
|
||||
algs/modes/gcm_generic.o \
|
||||
algs/modes/modes.o \
|
||||
algs/sha2/sha2_generic.o \
|
||||
algs/sha2/sha256_impl.o \
|
||||
algs/sha2/sha512_impl.o \
|
||||
algs/skein/skein.o \
|
||||
algs/skein/skein_block.o \
|
||||
algs/skein/skein_iv.o \
|
||||
@ -119,30 +121,40 @@ ICP_OBJS_X86_64 := \
|
||||
asm-x86_64/blake3/blake3_avx512.o \
|
||||
asm-x86_64/blake3/blake3_sse2.o \
|
||||
asm-x86_64/blake3/blake3_sse41.o \
|
||||
asm-x86_64/sha2/sha256-x86_64.o \
|
||||
asm-x86_64/sha2/sha512-x86_64.o \
|
||||
asm-x86_64/modes/aesni-gcm-x86_64.o \
|
||||
asm-x86_64/modes/gcm_pclmulqdq.o \
|
||||
asm-x86_64/modes/ghash-x86_64.o
|
||||
|
||||
|
||||
ICP_OBJS_X86 := \
|
||||
algs/aes/aes_impl_aesni.o \
|
||||
algs/aes/aes_impl_x86-64.o \
|
||||
algs/modes/gcm_pclmulqdq.o
|
||||
|
||||
ICP_OBJS_ARM := \
|
||||
asm-arm/sha2/sha256-armv7.o \
|
||||
asm-arm/sha2/sha512-armv7.o
|
||||
|
||||
ICP_OBJS_ARM64 := \
|
||||
asm-aarch64/blake3/b3_aarch64_sse2.o \
|
||||
asm-aarch64/blake3/b3_aarch64_sse41.o
|
||||
|
||||
asm-aarch64/blake3/b3_aarch64_sse41.o \
|
||||
asm-aarch64/sha2/sha256-armv8.o \
|
||||
asm-aarch64/sha2/sha512-armv8.o
|
||||
|
||||
ICP_OBJS_PPC_PPC64 := \
|
||||
asm-ppc64/blake3/b3_ppc64le_sse2.o \
|
||||
asm-ppc64/blake3/b3_ppc64le_sse41.o
|
||||
asm-ppc64/blake3/b3_ppc64le_sse41.o \
|
||||
asm-ppc64/sha2/sha256-p8.o \
|
||||
asm-ppc64/sha2/sha512-p8.o \
|
||||
asm-ppc64/sha2/sha256-ppc.o \
|
||||
asm-ppc64/sha2/sha512-ppc.o
|
||||
|
||||
zfs-objs += $(addprefix icp/,$(ICP_OBJS))
|
||||
zfs-$(CONFIG_X86) += $(addprefix icp/,$(ICP_OBJS_X86))
|
||||
zfs-$(CONFIG_UML_X86)+= $(addprefix icp/,$(ICP_OBJS_X86))
|
||||
zfs-$(CONFIG_X86_64) += $(addprefix icp/,$(ICP_OBJS_X86_64))
|
||||
zfs-$(CONFIG_ARM) += $(addprefix icp/,$(ICP_OBJS_ARM))
|
||||
zfs-$(CONFIG_ARM64) += $(addprefix icp/,$(ICP_OBJS_ARM64))
|
||||
zfs-$(CONFIG_PPC) += $(addprefix icp/,$(ICP_OBJS_PPC_PPC64))
|
||||
zfs-$(CONFIG_PPC64) += $(addprefix icp/,$(ICP_OBJS_PPC_PPC64))
|
||||
@ -156,6 +168,11 @@ $(addprefix $(obj)/icp/,$(ICP_OBJS) $(ICP_OBJS_X86) $(ICP_OBJS_X86_64) \
|
||||
# Suppress objtool "return with modified stack frame" warnings.
|
||||
OBJECT_FILES_NON_STANDARD_aesni-gcm-x86_64.o := y
|
||||
|
||||
# Suppress objtool "unsupported stack pointer realignment" warnings.
|
||||
# See #6950 for the reasoning.
|
||||
OBJECT_FILES_NON_STANDARD_sha256-x86_64.o := y
|
||||
OBJECT_FILES_NON_STANDARD_sha512-x86_64.o := y
|
||||
|
||||
LUA_OBJS := \
|
||||
lapi.o \
|
||||
lauxlib.o \
|
||||
@ -382,6 +399,7 @@ ZFS_OBJS := \
|
||||
zfs_chksum.o \
|
||||
zfs_fm.o \
|
||||
zfs_fuid.o \
|
||||
zfs_impl.o \
|
||||
zfs_ioctl.o \
|
||||
zfs_log.o \
|
||||
zfs_onexit.o \
|
||||
|
@ -13,10 +13,15 @@ KMOD= openzfs
|
||||
${SRCDIR}/lua \
|
||||
${SRCDIR}/nvpair \
|
||||
${SRCDIR}/icp/algs/blake3 \
|
||||
${SRCDIR}/icp/algs/edonr \
|
||||
${SRCDIR}/icp/algs/sha2 \
|
||||
${SRCDIR}/icp/asm-aarch64/blake3 \
|
||||
${SRCDIR}/icp/asm-aarch64/sha2 \
|
||||
${SRCDIR}/icp/asm-arm/sha2 \
|
||||
${SRCDIR}/icp/asm-ppc64/sha2 \
|
||||
${SRCDIR}/icp/asm-ppc64/blake3 \
|
||||
${SRCDIR}/icp/asm-x86_64/blake3 \
|
||||
${SRCDIR}/icp/algs/edonr \
|
||||
${SRCDIR}/icp/asm-x86_64/sha2 \
|
||||
${SRCDIR}/os/freebsd/spl \
|
||||
${SRCDIR}/os/freebsd/zfs \
|
||||
${SRCDIR}/unicode \
|
||||
@ -27,8 +32,6 @@ KMOD= openzfs
|
||||
${SRCDIR}/zstd/lib/compress \
|
||||
${SRCDIR}/zstd/lib/decompress
|
||||
|
||||
|
||||
|
||||
CFLAGS+= -I${INCDIR}
|
||||
CFLAGS+= -I${INCDIR}/os/freebsd
|
||||
CFLAGS+= -I${INCDIR}/os/freebsd/spl
|
||||
@ -88,8 +91,7 @@ SRCS+= edonr.c
|
||||
#icp/algs/blake3
|
||||
SRCS+= blake3.c \
|
||||
blake3_generic.c \
|
||||
blake3_impl.c \
|
||||
blake3_x86-64.c
|
||||
blake3_impl.c
|
||||
|
||||
#icp/asm-aarch64/blake3
|
||||
SRCS+= b3_aarch64_sse2.S \
|
||||
@ -105,6 +107,29 @@ SRCS+= blake3_avx2.S \
|
||||
blake3_sse2.S \
|
||||
blake3_sse41.S
|
||||
|
||||
#icp/algs/sha2
|
||||
SRCS+= sha2_generic.c \
|
||||
sha256_impl.c \
|
||||
sha512_impl.c
|
||||
|
||||
#icp/asm-arm/sha2
|
||||
SRCS+= sha256-armv7.S \
|
||||
sha512-armv7.S
|
||||
|
||||
#icp/asm-aarch64/sha2
|
||||
SRCS+= sha256-armv8.S \
|
||||
sha512-armv8.S
|
||||
|
||||
#icp/asm-ppc64/sha2
|
||||
SRCS+= sha256-p8.S \
|
||||
sha512-p8.S \
|
||||
sha256-ppc.S \
|
||||
sha512-ppc.S
|
||||
|
||||
#icp/asm-x86_64/sha2
|
||||
SRCS+= sha256-x86_64.S \
|
||||
sha512-x86_64.S
|
||||
|
||||
#lua
|
||||
SRCS+= lapi.c \
|
||||
lauxlib.c \
|
||||
@ -320,6 +345,7 @@ SRCS+= abd.c \
|
||||
zfs_file_os.c \
|
||||
zfs_fm.c \
|
||||
zfs_fuid.c \
|
||||
zfs_impl.c \
|
||||
zfs_ioctl.c \
|
||||
zfs_log.c \
|
||||
zfs_onexit.c \
|
||||
|
299
module/icp/algs/sha2/sha256_impl.c
Normal file
299
module/icp/algs/sha2/sha256_impl.c
Normal file
@ -0,0 +1,299 @@
|
||||
/*
|
||||
* CDDL HEADER START
|
||||
*
|
||||
* The contents of this file are subject to the terms of the
|
||||
* Common Development and Distribution License (the "License").
|
||||
* You may not use this file except in compliance with the License.
|
||||
*
|
||||
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
||||
* or https://opensource.org/licenses/CDDL-1.0.
|
||||
* See the License for the specific language governing permissions
|
||||
* and limitations under the License.
|
||||
*
|
||||
* When distributing Covered Code, include this CDDL HEADER in each
|
||||
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
||||
* If applicable, add the following below this CDDL HEADER, with the
|
||||
* fields enclosed by brackets "[]" replaced with your own identifying
|
||||
* information: Portions Copyright [yyyy] [name of copyright owner]
|
||||
*
|
||||
* CDDL HEADER END
|
||||
*/
|
||||
|
||||
/*
|
||||
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
|
||||
*/
|
||||
|
||||
#include <sys/zfs_context.h>
|
||||
#include <sys/zfs_impl.h>
|
||||
#include <sys/sha2.h>
|
||||
#include <sys/simd.h>
|
||||
|
||||
#include <sha2/sha2_impl.h>
|
||||
|
||||
#define TF(E, N) \
|
||||
extern void E(uint32_t s[8], const void *, size_t); \
|
||||
static inline void N(uint32_t s[8], const void *d, size_t b) { \
|
||||
kfpu_begin(); E(s, d, b); kfpu_end(); \
|
||||
}
|
||||
|
||||
/* some implementation is always okay */
|
||||
static inline boolean_t sha2_is_supported(void)
|
||||
{
|
||||
return (B_TRUE);
|
||||
}
|
||||
|
||||
#if defined(__x86_64)
|
||||
|
||||
extern void zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t);
|
||||
const sha256_ops_t sha256_x64_impl = {
|
||||
.is_supported = sha2_is_supported,
|
||||
.transform = zfs_sha256_transform_x64,
|
||||
.name = "x64"
|
||||
};
|
||||
|
||||
#if defined(HAVE_SSSE3)
|
||||
static boolean_t sha2_have_ssse3(void)
|
||||
{
|
||||
return (kfpu_allowed() && zfs_ssse3_available());
|
||||
}
|
||||
|
||||
TF(zfs_sha256_transform_ssse3, tf_sha256_ssse3);
|
||||
const sha256_ops_t sha256_ssse3_impl = {
|
||||
.is_supported = sha2_have_ssse3,
|
||||
.transform = tf_sha256_ssse3,
|
||||
.name = "ssse3"
|
||||
};
|
||||
#endif
|
||||
|
||||
#if defined(HAVE_AVX)
|
||||
static boolean_t sha2_have_avx(void)
|
||||
{
|
||||
return (kfpu_allowed() && zfs_avx_available());
|
||||
}
|
||||
|
||||
TF(zfs_sha256_transform_avx, tf_sha256_avx);
|
||||
const sha256_ops_t sha256_avx_impl = {
|
||||
.is_supported = sha2_have_avx,
|
||||
.transform = tf_sha256_avx,
|
||||
.name = "avx"
|
||||
};
|
||||
#endif
|
||||
|
||||
#if defined(HAVE_AVX2)
|
||||
static boolean_t sha2_have_avx2(void)
|
||||
{
|
||||
return (kfpu_allowed() && zfs_avx2_available());
|
||||
}
|
||||
|
||||
TF(zfs_sha256_transform_avx2, tf_sha256_avx2);
|
||||
const sha256_ops_t sha256_avx2_impl = {
|
||||
.is_supported = sha2_have_avx2,
|
||||
.transform = tf_sha256_avx2,
|
||||
.name = "avx2"
|
||||
};
|
||||
#endif
|
||||
|
||||
#if defined(HAVE_SSE4_1)
|
||||
static boolean_t sha2_have_shani(void)
|
||||
{
|
||||
return (kfpu_allowed() && zfs_sse4_1_available() && \
|
||||
zfs_shani_available());
|
||||
}
|
||||
|
||||
TF(zfs_sha256_transform_shani, tf_sha256_shani);
|
||||
const sha256_ops_t sha256_shani_impl = {
|
||||
.is_supported = sha2_have_shani,
|
||||
.transform = tf_sha256_shani,
|
||||
.name = "shani"
|
||||
};
|
||||
#endif
|
||||
|
||||
#elif defined(__aarch64__) || defined(__arm__)
|
||||
static boolean_t sha256_have_neon(void)
|
||||
{
|
||||
return (kfpu_allowed() && zfs_neon_available());
|
||||
}
|
||||
|
||||
static boolean_t sha256_have_armv8ce(void)
|
||||
{
|
||||
return (kfpu_allowed() && zfs_sha256_available());
|
||||
}
|
||||
|
||||
extern void zfs_sha256_block_armv7(uint32_t s[8], const void *, size_t);
|
||||
const sha256_ops_t sha256_armv7_impl = {
|
||||
.is_supported = sha2_is_supported,
|
||||
.transform = zfs_sha256_block_armv7,
|
||||
.name = "armv7"
|
||||
};
|
||||
|
||||
TF(zfs_sha256_block_neon, tf_sha256_neon);
|
||||
const sha256_ops_t sha256_neon_impl = {
|
||||
.is_supported = sha256_have_neon,
|
||||
.transform = tf_sha256_neon,
|
||||
.name = "neon"
|
||||
};
|
||||
|
||||
TF(zfs_sha256_block_armv8, tf_sha256_armv8ce);
|
||||
const sha256_ops_t sha256_armv8_impl = {
|
||||
.is_supported = sha256_have_armv8ce,
|
||||
.transform = tf_sha256_armv8ce,
|
||||
.name = "armv8-ce"
|
||||
};
|
||||
|
||||
#elif defined(__PPC64__)
|
||||
static boolean_t sha256_have_vsx(void)
|
||||
{
|
||||
return (kfpu_allowed() && zfs_vsx_available());
|
||||
}
|
||||
|
||||
TF(zfs_sha256_ppc, tf_sha256_ppc);
|
||||
const sha256_ops_t sha256_ppc_impl = {
|
||||
.is_supported = sha2_is_supported,
|
||||
.transform = tf_sha256_ppc,
|
||||
.name = "ppc"
|
||||
};
|
||||
|
||||
TF(zfs_sha256_power8, tf_sha256_power8);
|
||||
const sha256_ops_t sha256_power8_impl = {
|
||||
.is_supported = sha256_have_vsx,
|
||||
.transform = tf_sha256_power8,
|
||||
.name = "power8"
|
||||
};
|
||||
#endif /* __PPC64__ */
|
||||
|
||||
/* the two generic ones */
|
||||
extern const sha256_ops_t sha256_generic_impl;
|
||||
|
||||
/* array with all sha256 implementations */
|
||||
static const sha256_ops_t *const sha256_impls[] = {
|
||||
&sha256_generic_impl,
|
||||
#if defined(__x86_64)
|
||||
&sha256_x64_impl,
|
||||
#endif
|
||||
#if defined(__x86_64) && defined(HAVE_SSSE3)
|
||||
&sha256_ssse3_impl,
|
||||
#endif
|
||||
#if defined(__x86_64) && defined(HAVE_AVX)
|
||||
&sha256_avx_impl,
|
||||
#endif
|
||||
#if defined(__x86_64) && defined(HAVE_AVX2)
|
||||
&sha256_avx2_impl,
|
||||
#endif
|
||||
#if defined(__x86_64) && defined(HAVE_SSE4_1)
|
||||
&sha256_shani_impl,
|
||||
#endif
|
||||
#if defined(__aarch64__) || defined(__arm__)
|
||||
&sha256_armv7_impl,
|
||||
&sha256_neon_impl,
|
||||
&sha256_armv8_impl,
|
||||
#endif
|
||||
#if defined(__PPC64__)
|
||||
&sha256_ppc_impl,
|
||||
&sha256_power8_impl,
|
||||
#endif /* __PPC64__ */
|
||||
};
|
||||
|
||||
/* use the generic implementation functions */
|
||||
#define IMPL_NAME "sha256"
|
||||
#define IMPL_OPS_T sha256_ops_t
|
||||
#define IMPL_ARRAY sha256_impls
|
||||
#define IMPL_GET_OPS sha256_get_ops
|
||||
#define ZFS_IMPL_OPS zfs_sha256_ops
|
||||
#include <generic_impl.c>
|
||||
|
||||
#ifdef _KERNEL
|
||||
|
||||
#define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
|
||||
|
||||
#if defined(__linux__)
|
||||
|
||||
static int
|
||||
sha256_param_get(char *buffer, zfs_kernel_param_t *unused)
|
||||
{
|
||||
const uint32_t impl = IMPL_READ(generic_impl_chosen);
|
||||
char *fmt;
|
||||
int cnt = 0;
|
||||
|
||||
/* cycling */
|
||||
fmt = IMPL_FMT(impl, IMPL_CYCLE);
|
||||
cnt += sprintf(buffer + cnt, fmt, "cycle");
|
||||
|
||||
/* list fastest */
|
||||
fmt = IMPL_FMT(impl, IMPL_FASTEST);
|
||||
cnt += sprintf(buffer + cnt, fmt, "fastest");
|
||||
|
||||
/* list all supported implementations */
|
||||
generic_impl_init();
|
||||
for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
|
||||
fmt = IMPL_FMT(impl, i);
|
||||
cnt += sprintf(buffer + cnt, fmt,
|
||||
generic_supp_impls[i]->name);
|
||||
}
|
||||
|
||||
return (cnt);
|
||||
}
|
||||
|
||||
static int
|
||||
sha256_param_set(const char *val, zfs_kernel_param_t *unused)
|
||||
{
|
||||
(void) unused;
|
||||
return (generic_impl_setname(val));
|
||||
}
|
||||
|
||||
#elif defined(__FreeBSD__)
|
||||
|
||||
#include <sys/sbuf.h>
|
||||
|
||||
static int
|
||||
sha256_param(ZFS_MODULE_PARAM_ARGS)
|
||||
{
|
||||
int err;
|
||||
|
||||
generic_impl_init();
|
||||
if (req->newptr == NULL) {
|
||||
const uint32_t impl = IMPL_READ(generic_impl_chosen);
|
||||
const int init_buflen = 64;
|
||||
const char *fmt;
|
||||
struct sbuf *s;
|
||||
|
||||
s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
|
||||
|
||||
/* cycling */
|
||||
fmt = IMPL_FMT(impl, IMPL_CYCLE);
|
||||
(void) sbuf_printf(s, fmt, "cycle");
|
||||
|
||||
/* list fastest */
|
||||
fmt = IMPL_FMT(impl, IMPL_FASTEST);
|
||||
(void) sbuf_printf(s, fmt, "fastest");
|
||||
|
||||
/* list all supported implementations */
|
||||
for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
|
||||
fmt = IMPL_FMT(impl, i);
|
||||
(void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
|
||||
}
|
||||
|
||||
err = sbuf_finish(s);
|
||||
sbuf_delete(s);
|
||||
|
||||
return (err);
|
||||
}
|
||||
|
||||
char buf[16];
|
||||
|
||||
err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
|
||||
if (err) {
|
||||
return (err);
|
||||
}
|
||||
|
||||
return (-generic_impl_setname(buf));
|
||||
}
|
||||
#endif
|
||||
|
||||
#undef IMPL_FMT
|
||||
|
||||
ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha256_impl,
|
||||
sha256_param_set, sha256_param_get, ZMOD_RW, \
|
||||
"Select SHA256 implementation.");
|
||||
#endif
|
||||
|
||||
#undef TF
|
562
module/icp/algs/sha2/sha2_generic.c
Normal file
562
module/icp/algs/sha2/sha2_generic.c
Normal file
@ -0,0 +1,562 @@
|
||||
/*
|
||||
* CDDL HEADER START
|
||||
*
|
||||
* The contents of this file are subject to the terms of the
|
||||
* Common Development and Distribution License (the "License").
|
||||
* You may not use this file except in compliance with the License.
|
||||
*
|
||||
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
||||
* or https://opensource.org/licenses/CDDL-1.0.
|
||||
* See the License for the specific language governing permissions
|
||||
* and limitations under the License.
|
||||
*
|
||||
* When distributing Covered Code, include this CDDL HEADER in each
|
||||
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
||||
* If applicable, add the following below this CDDL HEADER, with the
|
||||
* fields enclosed by brackets "[]" replaced with your own identifying
|
||||
* information: Portions Copyright [yyyy] [name of copyright owner]
|
||||
*
|
||||
* CDDL HEADER END
|
||||
*/
|
||||
|
||||
/*
|
||||
* Based on public domain code in cppcrypto 0.10.
|
||||
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
|
||||
*/
|
||||
|
||||
#include <sys/zfs_context.h>
|
||||
#include <sys/zfs_impl.h>
|
||||
#include <sys/sha2.h>
|
||||
|
||||
#include <sha2/sha2_impl.h>
|
||||
|
||||
/*
|
||||
* On i386, gcc brings this for sha512_generic():
|
||||
* error: the frame size of 1040 bytes is larger than 1024
|
||||
*/
|
||||
#if defined(__GNUC__) && defined(_ILP32)
|
||||
#pragma GCC diagnostic ignored "-Wframe-larger-than="
|
||||
#endif
|
||||
|
||||
/* SHA256 */
|
||||
static const uint32_t SHA256_K[64] = {
|
||||
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
|
||||
0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
|
||||
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
|
||||
0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
|
||||
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
|
||||
0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
|
||||
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
|
||||
0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
|
||||
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
|
||||
0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
|
||||
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
|
||||
0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
||||
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
|
||||
0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
|
||||
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
|
||||
0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2
|
||||
};
|
||||
|
||||
#define Ch(x, y, z) ((z) ^ ((x) & ((y) ^ (z))))
|
||||
#define Maj(x, y, z) (((y) & (z)) | (((y) | (z)) & (x)))
|
||||
|
||||
#define rotr32(x, n) (((x) >> n) | ((x) << (32 - n)))
|
||||
#define sum0(x) (rotr32((x), 2) ^ rotr32((x), 13) ^ rotr32((x), 22))
|
||||
#define sum1(x) (rotr32((x), 6) ^ rotr32((x), 11) ^ rotr32((x), 25))
|
||||
#define sigma0(x) (rotr32((x), 7) ^ rotr32((x), 18) ^ ((x) >> 3))
|
||||
#define sigma1(x) (rotr32((x), 17) ^ rotr32((x), 19) ^ ((x) >> 10))
|
||||
|
||||
#define WU(j) (W[j & 15] += sigma1(W[(j + 14) & 15]) \
|
||||
+ W[(j + 9) & 15] + sigma0(W[(j + 1) & 15]))
|
||||
|
||||
#define COMPRESS(i, j, K) \
|
||||
T1 = h + sum1(e) + Ch(e, f, g) + K[i + j] + (i? WU(j): W[j]); \
|
||||
T2 = sum0(a) + Maj(a, b, c); \
|
||||
h = g, g = f, f = e, e = d + T1; \
|
||||
d = c, c = b, b = a, a = T1 + T2;
|
||||
|
||||
static void sha256_generic(uint32_t state[8], const void *data, size_t num_blks)
|
||||
{
|
||||
uint64_t blk;
|
||||
|
||||
for (blk = 0; blk < num_blks; blk++) {
|
||||
uint32_t W[16];
|
||||
uint32_t a, b, c, d, e, f, g, h;
|
||||
uint32_t T1, T2;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < 16; i++) {
|
||||
W[i] = BE_32( \
|
||||
(((const uint32_t *)(data))[blk * 16 + i]));
|
||||
}
|
||||
|
||||
a = state[0];
|
||||
b = state[1];
|
||||
c = state[2];
|
||||
d = state[3];
|
||||
e = state[4];
|
||||
f = state[5];
|
||||
g = state[6];
|
||||
h = state[7];
|
||||
|
||||
for (i = 0; i <= 63; i += 16) {
|
||||
COMPRESS(i, 0, SHA256_K);
|
||||
COMPRESS(i, 1, SHA256_K);
|
||||
COMPRESS(i, 2, SHA256_K);
|
||||
COMPRESS(i, 3, SHA256_K);
|
||||
COMPRESS(i, 4, SHA256_K);
|
||||
COMPRESS(i, 5, SHA256_K);
|
||||
COMPRESS(i, 6, SHA256_K);
|
||||
COMPRESS(i, 7, SHA256_K);
|
||||
COMPRESS(i, 8, SHA256_K);
|
||||
COMPRESS(i, 9, SHA256_K);
|
||||
COMPRESS(i, 10, SHA256_K);
|
||||
COMPRESS(i, 11, SHA256_K);
|
||||
COMPRESS(i, 12, SHA256_K);
|
||||
COMPRESS(i, 13, SHA256_K);
|
||||
COMPRESS(i, 14, SHA256_K);
|
||||
COMPRESS(i, 15, SHA256_K);
|
||||
}
|
||||
|
||||
state[0] += a;
|
||||
state[1] += b;
|
||||
state[2] += c;
|
||||
state[3] += d;
|
||||
state[4] += e;
|
||||
state[5] += f;
|
||||
state[6] += g;
|
||||
state[7] += h;
|
||||
}
|
||||
}
|
||||
|
||||
#undef sum0
|
||||
#undef sum1
|
||||
#undef sigma0
|
||||
#undef sigma1
|
||||
|
||||
#define rotr64(x, n) (((x) >> n) | ((x) << (64 - n)))
|
||||
#define sum0(x) (rotr64((x), 28) ^ rotr64((x), 34) ^ rotr64((x), 39))
|
||||
#define sum1(x) (rotr64((x), 14) ^ rotr64((x), 18) ^ rotr64((x), 41))
|
||||
#define sigma0(x) (rotr64((x), 1) ^ rotr64((x), 8) ^ ((x) >> 7))
|
||||
#define sigma1(x) (rotr64((x), 19) ^ rotr64((x), 61) ^ ((x) >> 6))
|
||||
|
||||
/* SHA512 */
|
||||
static const uint64_t SHA512_K[80] = {
|
||||
0x428a2f98d728ae22, 0x7137449123ef65cd, 0xb5c0fbcfec4d3b2f,
|
||||
0xe9b5dba58189dbbc, 0x3956c25bf348b538, 0x59f111f1b605d019,
|
||||
0x923f82a4af194f9b, 0xab1c5ed5da6d8118, 0xd807aa98a3030242,
|
||||
0x12835b0145706fbe, 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2,
|
||||
0x72be5d74f27b896f, 0x80deb1fe3b1696b1, 0x9bdc06a725c71235,
|
||||
0xc19bf174cf692694, 0xe49b69c19ef14ad2, 0xefbe4786384f25e3,
|
||||
0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65, 0x2de92c6f592b0275,
|
||||
0x4a7484aa6ea6e483, 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5,
|
||||
0x983e5152ee66dfab, 0xa831c66d2db43210, 0xb00327c898fb213f,
|
||||
0xbf597fc7beef0ee4, 0xc6e00bf33da88fc2, 0xd5a79147930aa725,
|
||||
0x06ca6351e003826f, 0x142929670a0e6e70, 0x27b70a8546d22ffc,
|
||||
0x2e1b21385c26c926, 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df,
|
||||
0x650a73548baf63de, 0x766a0abb3c77b2a8, 0x81c2c92e47edaee6,
|
||||
0x92722c851482353b, 0xa2bfe8a14cf10364, 0xa81a664bbc423001,
|
||||
0xc24b8b70d0f89791, 0xc76c51a30654be30, 0xd192e819d6ef5218,
|
||||
0xd69906245565a910, 0xf40e35855771202a, 0x106aa07032bbd1b8,
|
||||
0x19a4c116b8d2d0c8, 0x1e376c085141ab53, 0x2748774cdf8eeb99,
|
||||
0x34b0bcb5e19b48a8, 0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb,
|
||||
0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3, 0x748f82ee5defb2fc,
|
||||
0x78a5636f43172f60, 0x84c87814a1f0ab72, 0x8cc702081a6439ec,
|
||||
0x90befffa23631e28, 0xa4506cebde82bde9, 0xbef9a3f7b2c67915,
|
||||
0xc67178f2e372532b, 0xca273eceea26619c, 0xd186b8c721c0c207,
|
||||
0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178, 0x06f067aa72176fba,
|
||||
0x0a637dc5a2c898a6, 0x113f9804bef90dae, 0x1b710b35131c471b,
|
||||
0x28db77f523047d84, 0x32caab7b40c72493, 0x3c9ebe0a15c9bebc,
|
||||
0x431d67c49c100d4c, 0x4cc5d4becb3e42b6, 0x597f299cfc657e2a,
|
||||
0x5fcb6fab3ad6faec, 0x6c44198c4a475817
|
||||
};
|
||||
|
||||
static void sha512_generic(uint64_t state[8], const void *data, size_t num_blks)
|
||||
{
|
||||
uint64_t blk;
|
||||
|
||||
for (blk = 0; blk < num_blks; blk++) {
|
||||
uint64_t W[16];
|
||||
uint64_t a, b, c, d, e, f, g, h;
|
||||
uint64_t T1, T2;
|
||||
int i;
|
||||
|
||||
for (i = 0; i < 16; i++) {
|
||||
W[i] = BE_64( \
|
||||
(((const uint64_t *)(data))[blk * 16 + i]));
|
||||
}
|
||||
|
||||
a = state[0];
|
||||
b = state[1];
|
||||
c = state[2];
|
||||
d = state[3];
|
||||
e = state[4];
|
||||
f = state[5];
|
||||
g = state[6];
|
||||
h = state[7];
|
||||
|
||||
for (i = 0; i <= 79; i += 16) {
|
||||
COMPRESS(i, 0, SHA512_K);
|
||||
COMPRESS(i, 1, SHA512_K);
|
||||
COMPRESS(i, 2, SHA512_K);
|
||||
COMPRESS(i, 3, SHA512_K);
|
||||
COMPRESS(i, 4, SHA512_K);
|
||||
COMPRESS(i, 5, SHA512_K);
|
||||
COMPRESS(i, 6, SHA512_K);
|
||||
COMPRESS(i, 7, SHA512_K);
|
||||
COMPRESS(i, 8, SHA512_K);
|
||||
COMPRESS(i, 9, SHA512_K);
|
||||
COMPRESS(i, 10, SHA512_K);
|
||||
COMPRESS(i, 11, SHA512_K);
|
||||
COMPRESS(i, 12, SHA512_K);
|
||||
COMPRESS(i, 13, SHA512_K);
|
||||
COMPRESS(i, 14, SHA512_K);
|
||||
COMPRESS(i, 15, SHA512_K);
|
||||
}
|
||||
state[0] += a;
|
||||
state[1] += b;
|
||||
state[2] += c;
|
||||
state[3] += d;
|
||||
state[4] += e;
|
||||
state[5] += f;
|
||||
state[6] += g;
|
||||
state[7] += h;
|
||||
}
|
||||
}
|
||||
|
||||
static void sha256_update(sha256_ctx *ctx, const uint8_t *data, size_t len)
|
||||
{
|
||||
uint64_t pos = ctx->count[0];
|
||||
uint64_t total = ctx->count[1];
|
||||
uint8_t *m = ctx->wbuf;
|
||||
const sha256_ops_t *ops = ctx->ops;
|
||||
|
||||
if (pos && pos + len >= 64) {
|
||||
memcpy(m + pos, data, 64 - pos);
|
||||
ops->transform(ctx->state, m, 1);
|
||||
len -= 64 - pos;
|
||||
total += (64 - pos) * 8;
|
||||
data += 64 - pos;
|
||||
pos = 0;
|
||||
}
|
||||
|
||||
if (len >= 64) {
|
||||
uint32_t blocks = len / 64;
|
||||
uint32_t bytes = blocks * 64;
|
||||
ops->transform(ctx->state, data, blocks);
|
||||
len -= bytes;
|
||||
total += (bytes) * 8;
|
||||
data += bytes;
|
||||
}
|
||||
memcpy(m + pos, data, len);
|
||||
|
||||
pos += len;
|
||||
total += len * 8;
|
||||
ctx->count[0] = pos;
|
||||
ctx->count[1] = total;
|
||||
}
|
||||
|
||||
static void sha512_update(sha512_ctx *ctx, const uint8_t *data, size_t len)
|
||||
{
|
||||
uint64_t pos = ctx->count[0];
|
||||
uint64_t total = ctx->count[1];
|
||||
uint8_t *m = ctx->wbuf;
|
||||
const sha512_ops_t *ops = ctx->ops;
|
||||
|
||||
if (pos && pos + len >= 128) {
|
||||
memcpy(m + pos, data, 128 - pos);
|
||||
ops->transform(ctx->state, m, 1);
|
||||
len -= 128 - pos;
|
||||
total += (128 - pos) * 8;
|
||||
data += 128 - pos;
|
||||
pos = 0;
|
||||
}
|
||||
|
||||
if (len >= 128) {
|
||||
uint64_t blocks = len / 128;
|
||||
uint64_t bytes = blocks * 128;
|
||||
ops->transform(ctx->state, data, blocks);
|
||||
len -= bytes;
|
||||
total += (bytes) * 8;
|
||||
data += bytes;
|
||||
}
|
||||
memcpy(m + pos, data, len);
|
||||
|
||||
pos += len;
|
||||
total += len * 8;
|
||||
ctx->count[0] = pos;
|
||||
ctx->count[1] = total;
|
||||
}
|
||||
|
||||
static void sha256_final(sha256_ctx *ctx, uint8_t *result, int bits)
|
||||
{
|
||||
uint64_t mlen, pos = ctx->count[0];
|
||||
uint8_t *m = ctx->wbuf;
|
||||
uint32_t *R = (uint32_t *)result;
|
||||
const sha256_ops_t *ops = ctx->ops;
|
||||
|
||||
m[pos++] = 0x80;
|
||||
if (pos > 56) {
|
||||
memset(m + pos, 0, 64 - pos);
|
||||
ops->transform(ctx->state, m, 1);
|
||||
pos = 0;
|
||||
}
|
||||
|
||||
memset(m + pos, 0, 64 - pos);
|
||||
mlen = BE_64(ctx->count[1]);
|
||||
memcpy(m + (64 - 8), &mlen, 64 / 8);
|
||||
ops->transform(ctx->state, m, 1);
|
||||
|
||||
switch (bits) {
|
||||
case 224: /* 28 - unused currently /TR */
|
||||
R[0] = BE_32(ctx->state[0]);
|
||||
R[1] = BE_32(ctx->state[1]);
|
||||
R[2] = BE_32(ctx->state[2]);
|
||||
R[3] = BE_32(ctx->state[3]);
|
||||
R[4] = BE_32(ctx->state[4]);
|
||||
R[5] = BE_32(ctx->state[5]);
|
||||
R[6] = BE_32(ctx->state[6]);
|
||||
break;
|
||||
case 256: /* 32 */
|
||||
R[0] = BE_32(ctx->state[0]);
|
||||
R[1] = BE_32(ctx->state[1]);
|
||||
R[2] = BE_32(ctx->state[2]);
|
||||
R[3] = BE_32(ctx->state[3]);
|
||||
R[4] = BE_32(ctx->state[4]);
|
||||
R[5] = BE_32(ctx->state[5]);
|
||||
R[6] = BE_32(ctx->state[6]);
|
||||
R[7] = BE_32(ctx->state[7]);
|
||||
break;
|
||||
}
|
||||
|
||||
memset(ctx, 0, sizeof (*ctx));
|
||||
}
|
||||
|
||||
static void sha512_final(sha512_ctx *ctx, uint8_t *result, int bits)
|
||||
{
|
||||
uint64_t mlen, pos = ctx->count[0];
|
||||
uint8_t *m = ctx->wbuf, *r;
|
||||
uint64_t *R = (uint64_t *)result;
|
||||
const sha512_ops_t *ops = ctx->ops;
|
||||
|
||||
m[pos++] = 0x80;
|
||||
if (pos > 112) {
|
||||
memset(m + pos, 0, 128 - pos);
|
||||
ops->transform(ctx->state, m, 1);
|
||||
pos = 0;
|
||||
}
|
||||
|
||||
memset(m + pos, 0, 128 - pos);
|
||||
mlen = BE_64(ctx->count[1]);
|
||||
memcpy(m + (128 - 8), &mlen, 64 / 8);
|
||||
ops->transform(ctx->state, m, 1);
|
||||
|
||||
switch (bits) {
|
||||
case 224: /* 28 => 3,5 x 8 */
|
||||
r = result + 24;
|
||||
R[0] = BE_64(ctx->state[0]);
|
||||
R[1] = BE_64(ctx->state[1]);
|
||||
R[2] = BE_64(ctx->state[2]);
|
||||
/* last 4 bytes are special here */
|
||||
*r++ = (uint8_t)(ctx->state[3] >> 56);
|
||||
*r++ = (uint8_t)(ctx->state[3] >> 48);
|
||||
*r++ = (uint8_t)(ctx->state[3] >> 40);
|
||||
*r++ = (uint8_t)(ctx->state[3] >> 32);
|
||||
break;
|
||||
case 256: /* 32 */
|
||||
R[0] = BE_64(ctx->state[0]);
|
||||
R[1] = BE_64(ctx->state[1]);
|
||||
R[2] = BE_64(ctx->state[2]);
|
||||
R[3] = BE_64(ctx->state[3]);
|
||||
break;
|
||||
case 384: /* 48 */
|
||||
R[0] = BE_64(ctx->state[0]);
|
||||
R[1] = BE_64(ctx->state[1]);
|
||||
R[2] = BE_64(ctx->state[2]);
|
||||
R[3] = BE_64(ctx->state[3]);
|
||||
R[4] = BE_64(ctx->state[4]);
|
||||
R[5] = BE_64(ctx->state[5]);
|
||||
break;
|
||||
case 512: /* 64 */
|
||||
R[0] = BE_64(ctx->state[0]);
|
||||
R[1] = BE_64(ctx->state[1]);
|
||||
R[2] = BE_64(ctx->state[2]);
|
||||
R[3] = BE_64(ctx->state[3]);
|
||||
R[4] = BE_64(ctx->state[4]);
|
||||
R[5] = BE_64(ctx->state[5]);
|
||||
R[6] = BE_64(ctx->state[6]);
|
||||
R[7] = BE_64(ctx->state[7]);
|
||||
break;
|
||||
}
|
||||
|
||||
memset(ctx, 0, sizeof (*ctx));
|
||||
}
|
||||
|
||||
/* SHA2 Init function */
|
||||
void
|
||||
SHA2Init(int algotype, SHA2_CTX *ctx)
|
||||
{
|
||||
sha256_ctx *ctx256 = &ctx->sha256;
|
||||
sha512_ctx *ctx512 = &ctx->sha512;
|
||||
|
||||
ASSERT3U(algotype, >=, SHA256_MECH_INFO_TYPE);
|
||||
ASSERT3U(algotype, <=, SHA512_256_MECH_INFO_TYPE);
|
||||
|
||||
memset(ctx, 0, sizeof (*ctx));
|
||||
ctx->algotype = algotype;
|
||||
switch (ctx->algotype) {
|
||||
case SHA256_MECH_INFO_TYPE:
|
||||
case SHA256_HMAC_MECH_INFO_TYPE:
|
||||
case SHA256_HMAC_GEN_MECH_INFO_TYPE:
|
||||
ctx256->state[0] = 0x6a09e667;
|
||||
ctx256->state[1] = 0xbb67ae85;
|
||||
ctx256->state[2] = 0x3c6ef372;
|
||||
ctx256->state[3] = 0xa54ff53a;
|
||||
ctx256->state[4] = 0x510e527f;
|
||||
ctx256->state[5] = 0x9b05688c;
|
||||
ctx256->state[6] = 0x1f83d9ab;
|
||||
ctx256->state[7] = 0x5be0cd19;
|
||||
ctx256->count[0] = 0;
|
||||
ctx256->ops = sha256_get_ops();
|
||||
break;
|
||||
case SHA384_MECH_INFO_TYPE:
|
||||
case SHA384_HMAC_MECH_INFO_TYPE:
|
||||
case SHA384_HMAC_GEN_MECH_INFO_TYPE:
|
||||
ctx512->state[0] = 0xcbbb9d5dc1059ed8ULL;
|
||||
ctx512->state[1] = 0x629a292a367cd507ULL;
|
||||
ctx512->state[2] = 0x9159015a3070dd17ULL;
|
||||
ctx512->state[3] = 0x152fecd8f70e5939ULL;
|
||||
ctx512->state[4] = 0x67332667ffc00b31ULL;
|
||||
ctx512->state[5] = 0x8eb44a8768581511ULL;
|
||||
ctx512->state[6] = 0xdb0c2e0d64f98fa7ULL;
|
||||
ctx512->state[7] = 0x47b5481dbefa4fa4ULL;
|
||||
ctx512->count[0] = 0;
|
||||
ctx512->count[1] = 0;
|
||||
ctx512->ops = sha512_get_ops();
|
||||
break;
|
||||
case SHA512_MECH_INFO_TYPE:
|
||||
case SHA512_HMAC_MECH_INFO_TYPE:
|
||||
case SHA512_HMAC_GEN_MECH_INFO_TYPE:
|
||||
ctx512->state[0] = 0x6a09e667f3bcc908ULL;
|
||||
ctx512->state[1] = 0xbb67ae8584caa73bULL;
|
||||
ctx512->state[2] = 0x3c6ef372fe94f82bULL;
|
||||
ctx512->state[3] = 0xa54ff53a5f1d36f1ULL;
|
||||
ctx512->state[4] = 0x510e527fade682d1ULL;
|
||||
ctx512->state[5] = 0x9b05688c2b3e6c1fULL;
|
||||
ctx512->state[6] = 0x1f83d9abfb41bd6bULL;
|
||||
ctx512->state[7] = 0x5be0cd19137e2179ULL;
|
||||
ctx512->count[0] = 0;
|
||||
ctx512->count[1] = 0;
|
||||
ctx512->ops = sha512_get_ops();
|
||||
break;
|
||||
case SHA512_224_MECH_INFO_TYPE:
|
||||
ctx512->state[0] = 0x8c3d37c819544da2ULL;
|
||||
ctx512->state[1] = 0x73e1996689dcd4d6ULL;
|
||||
ctx512->state[2] = 0x1dfab7ae32ff9c82ULL;
|
||||
ctx512->state[3] = 0x679dd514582f9fcfULL;
|
||||
ctx512->state[4] = 0x0f6d2b697bd44da8ULL;
|
||||
ctx512->state[5] = 0x77e36f7304c48942ULL;
|
||||
ctx512->state[6] = 0x3f9d85a86a1d36c8ULL;
|
||||
ctx512->state[7] = 0x1112e6ad91d692a1ULL;
|
||||
ctx512->count[0] = 0;
|
||||
ctx512->count[1] = 0;
|
||||
ctx512->ops = sha512_get_ops();
|
||||
break;
|
||||
case SHA512_256_MECH_INFO_TYPE:
|
||||
ctx512->state[0] = 0x22312194fc2bf72cULL;
|
||||
ctx512->state[1] = 0x9f555fa3c84c64c2ULL;
|
||||
ctx512->state[2] = 0x2393b86b6f53b151ULL;
|
||||
ctx512->state[3] = 0x963877195940eabdULL;
|
||||
ctx512->state[4] = 0x96283ee2a88effe3ULL;
|
||||
ctx512->state[5] = 0xbe5e1e2553863992ULL;
|
||||
ctx512->state[6] = 0x2b0199fc2c85b8aaULL;
|
||||
ctx512->state[7] = 0x0eb72ddc81c52ca2ULL;
|
||||
ctx512->count[0] = 0;
|
||||
ctx512->count[1] = 0;
|
||||
ctx512->ops = sha512_get_ops();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* SHA2 Update function */
|
||||
void
|
||||
SHA2Update(SHA2_CTX *ctx, const void *data, size_t len)
|
||||
{
|
||||
/* check for zero input length */
|
||||
if (len == 0)
|
||||
return;
|
||||
|
||||
ASSERT3P(data, !=, NULL);
|
||||
|
||||
switch (ctx->algotype) {
|
||||
case SHA256_MECH_INFO_TYPE:
|
||||
case SHA256_HMAC_MECH_INFO_TYPE:
|
||||
case SHA256_HMAC_GEN_MECH_INFO_TYPE:
|
||||
sha256_update(&ctx->sha256, data, len);
|
||||
break;
|
||||
case SHA384_MECH_INFO_TYPE:
|
||||
case SHA384_HMAC_MECH_INFO_TYPE:
|
||||
case SHA384_HMAC_GEN_MECH_INFO_TYPE:
|
||||
sha512_update(&ctx->sha512, data, len);
|
||||
break;
|
||||
case SHA512_MECH_INFO_TYPE:
|
||||
case SHA512_HMAC_MECH_INFO_TYPE:
|
||||
case SHA512_HMAC_GEN_MECH_INFO_TYPE:
|
||||
sha512_update(&ctx->sha512, data, len);
|
||||
break;
|
||||
case SHA512_224_MECH_INFO_TYPE:
|
||||
sha512_update(&ctx->sha512, data, len);
|
||||
break;
|
||||
case SHA512_256_MECH_INFO_TYPE:
|
||||
sha512_update(&ctx->sha512, data, len);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* SHA2Final function */
|
||||
void
|
||||
SHA2Final(void *digest, SHA2_CTX *ctx)
|
||||
{
|
||||
switch (ctx->algotype) {
|
||||
case SHA256_MECH_INFO_TYPE:
|
||||
case SHA256_HMAC_MECH_INFO_TYPE:
|
||||
case SHA256_HMAC_GEN_MECH_INFO_TYPE:
|
||||
sha256_final(&ctx->sha256, digest, 256);
|
||||
break;
|
||||
case SHA384_MECH_INFO_TYPE:
|
||||
case SHA384_HMAC_MECH_INFO_TYPE:
|
||||
case SHA384_HMAC_GEN_MECH_INFO_TYPE:
|
||||
sha512_final(&ctx->sha512, digest, 384);
|
||||
break;
|
||||
case SHA512_MECH_INFO_TYPE:
|
||||
case SHA512_HMAC_MECH_INFO_TYPE:
|
||||
case SHA512_HMAC_GEN_MECH_INFO_TYPE:
|
||||
sha512_final(&ctx->sha512, digest, 512);
|
||||
break;
|
||||
case SHA512_224_MECH_INFO_TYPE:
|
||||
sha512_final(&ctx->sha512, digest, 224);
|
||||
break;
|
||||
case SHA512_256_MECH_INFO_TYPE:
|
||||
sha512_final(&ctx->sha512, digest, 256);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* the generic implementation is always okay */
|
||||
static boolean_t sha2_is_supported(void)
|
||||
{
|
||||
return (B_TRUE);
|
||||
}
|
||||
|
||||
const sha256_ops_t sha256_generic_impl = {
|
||||
.name = "generic",
|
||||
.transform = sha256_generic,
|
||||
.is_supported = sha2_is_supported
|
||||
};
|
||||
|
||||
const sha512_ops_t sha512_generic_impl = {
|
||||
.name = "generic",
|
||||
.transform = sha512_generic,
|
||||
.is_supported = sha2_is_supported
|
||||
};
|
276
module/icp/algs/sha2/sha512_impl.c
Normal file
276
module/icp/algs/sha2/sha512_impl.c
Normal file
@ -0,0 +1,276 @@
|
||||
/*
|
||||
* CDDL HEADER START
|
||||
*
|
||||
* The contents of this file are subject to the terms of the
|
||||
* Common Development and Distribution License (the "License").
|
||||
* You may not use this file except in compliance with the License.
|
||||
*
|
||||
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
||||
* or https://opensource.org/licenses/CDDL-1.0.
|
||||
* See the License for the specific language governing permissions
|
||||
* and limitations under the License.
|
||||
*
|
||||
* When distributing Covered Code, include this CDDL HEADER in each
|
||||
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
||||
* If applicable, add the following below this CDDL HEADER, with the
|
||||
* fields enclosed by brackets "[]" replaced with your own identifying
|
||||
* information: Portions Copyright [yyyy] [name of copyright owner]
|
||||
*
|
||||
* CDDL HEADER END
|
||||
*/
|
||||
|
||||
/*
|
||||
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
|
||||
*/
|
||||
|
||||
#include <sys/zfs_context.h>
|
||||
#include <sys/zfs_impl.h>
|
||||
#include <sys/sha2.h>
|
||||
#include <sys/simd.h>
|
||||
|
||||
#include <sha2/sha2_impl.h>
|
||||
|
||||
#define TF(E, N) \
|
||||
extern void E(uint64_t s[8], const void *, size_t); \
|
||||
static inline void N(uint64_t s[8], const void *d, size_t b) { \
|
||||
kfpu_begin(); E(s, d, b); kfpu_end(); \
|
||||
}
|
||||
|
||||
/* some implementation is always okay */
|
||||
static inline boolean_t sha2_is_supported(void)
|
||||
{
|
||||
return (B_TRUE);
|
||||
}
|
||||
|
||||
#if defined(__x86_64)
|
||||
|
||||
extern void zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t);
|
||||
const sha512_ops_t sha512_x64_impl = {
|
||||
.is_supported = sha2_is_supported,
|
||||
.transform = zfs_sha512_transform_x64,
|
||||
.name = "x64"
|
||||
};
|
||||
|
||||
#if defined(HAVE_AVX)
|
||||
static boolean_t sha2_have_avx(void)
|
||||
{
|
||||
return (kfpu_allowed() && zfs_avx_available());
|
||||
}
|
||||
|
||||
TF(zfs_sha512_transform_avx, tf_sha512_avx);
|
||||
const sha512_ops_t sha512_avx_impl = {
|
||||
.is_supported = sha2_have_avx,
|
||||
.transform = tf_sha512_avx,
|
||||
.name = "avx"
|
||||
};
|
||||
#endif
|
||||
|
||||
#if defined(HAVE_AVX2)
|
||||
static boolean_t sha2_have_avx2(void)
|
||||
{
|
||||
return (kfpu_allowed() && zfs_avx2_available());
|
||||
}
|
||||
|
||||
TF(zfs_sha512_transform_avx2, tf_sha512_avx2);
|
||||
const sha512_ops_t sha512_avx2_impl = {
|
||||
.is_supported = sha2_have_avx2,
|
||||
.transform = tf_sha512_avx2,
|
||||
.name = "avx2"
|
||||
};
|
||||
#endif
|
||||
|
||||
#elif defined(__aarch64__)
|
||||
extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t);
|
||||
const sha512_ops_t sha512_armv7_impl = {
|
||||
.is_supported = sha2_is_supported,
|
||||
.transform = zfs_sha512_block_armv7,
|
||||
.name = "armv7"
|
||||
};
|
||||
|
||||
static boolean_t sha512_have_armv8ce(void)
|
||||
{
|
||||
return (kfpu_allowed() && zfs_sha512_available());
|
||||
}
|
||||
|
||||
TF(zfs_sha512_block_armv8, tf_sha512_armv8ce);
|
||||
const sha512_ops_t sha512_armv8_impl = {
|
||||
.is_supported = sha512_have_armv8ce,
|
||||
.transform = tf_sha512_armv8ce,
|
||||
.name = "armv8-ce"
|
||||
};
|
||||
|
||||
#elif defined(__arm__)
|
||||
extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t);
|
||||
const sha512_ops_t sha512_armv7_impl = {
|
||||
.is_supported = sha2_is_supported,
|
||||
.transform = zfs_sha512_block_armv7,
|
||||
.name = "armv7"
|
||||
};
|
||||
|
||||
static boolean_t sha512_have_neon(void)
|
||||
{
|
||||
return (kfpu_allowed() && zfs_neon_available());
|
||||
}
|
||||
|
||||
TF(zfs_sha512_block_neon, tf_sha512_neon);
|
||||
const sha512_ops_t sha512_neon_impl = {
|
||||
.is_supported = sha512_have_neon,
|
||||
.transform = tf_sha512_neon,
|
||||
.name = "neon"
|
||||
};
|
||||
|
||||
#elif defined(__PPC64__)
|
||||
TF(zfs_sha512_ppc, tf_sha512_ppc);
|
||||
const sha512_ops_t sha512_ppc_impl = {
|
||||
.is_supported = sha2_is_supported,
|
||||
.transform = tf_sha512_ppc,
|
||||
.name = "ppc"
|
||||
};
|
||||
|
||||
static boolean_t sha512_have_vsx(void)
|
||||
{
|
||||
return (kfpu_allowed() && zfs_vsx_available());
|
||||
}
|
||||
|
||||
TF(zfs_sha512_power8, tf_sha512_power8);
|
||||
const sha512_ops_t sha512_power8_impl = {
|
||||
.is_supported = sha512_have_vsx,
|
||||
.transform = tf_sha512_power8,
|
||||
.name = "power8"
|
||||
};
|
||||
#endif /* __PPC64__ */
|
||||
|
||||
/* the two generic ones */
|
||||
extern const sha512_ops_t sha512_generic_impl;
|
||||
|
||||
/* array with all sha512 implementations */
|
||||
static const sha512_ops_t *const sha512_impls[] = {
|
||||
&sha512_generic_impl,
|
||||
#if defined(__x86_64)
|
||||
&sha512_x64_impl,
|
||||
#endif
|
||||
#if defined(__x86_64) && defined(HAVE_AVX)
|
||||
&sha512_avx_impl,
|
||||
#endif
|
||||
#if defined(__x86_64) && defined(HAVE_AVX2)
|
||||
&sha512_avx2_impl,
|
||||
#endif
|
||||
#if defined(__aarch64__)
|
||||
&sha512_armv7_impl,
|
||||
&sha512_armv8_impl,
|
||||
#endif
|
||||
#if defined(__arm__)
|
||||
&sha512_armv7_impl,
|
||||
&sha512_neon_impl,
|
||||
#endif
|
||||
#if defined(__PPC64__)
|
||||
&sha512_ppc_impl,
|
||||
&sha512_power8_impl,
|
||||
#endif /* __PPC64__ */
|
||||
};
|
||||
|
||||
/* use the generic implementation functions */
|
||||
#define IMPL_NAME "sha512"
|
||||
#define IMPL_OPS_T sha512_ops_t
|
||||
#define IMPL_ARRAY sha512_impls
|
||||
#define IMPL_GET_OPS sha512_get_ops
|
||||
#define ZFS_IMPL_OPS zfs_sha512_ops
|
||||
#include <generic_impl.c>
|
||||
|
||||
#ifdef _KERNEL
|
||||
|
||||
#define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
|
||||
|
||||
#if defined(__linux__)
|
||||
|
||||
static int
|
||||
sha512_param_get(char *buffer, zfs_kernel_param_t *unused)
|
||||
{
|
||||
const uint32_t impl = IMPL_READ(generic_impl_chosen);
|
||||
char *fmt;
|
||||
int cnt = 0;
|
||||
|
||||
/* cycling */
|
||||
fmt = IMPL_FMT(impl, IMPL_CYCLE);
|
||||
cnt += sprintf(buffer + cnt, fmt, "cycle");
|
||||
|
||||
/* list fastest */
|
||||
fmt = IMPL_FMT(impl, IMPL_FASTEST);
|
||||
cnt += sprintf(buffer + cnt, fmt, "fastest");
|
||||
|
||||
/* list all supported implementations */
|
||||
generic_impl_init();
|
||||
for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
|
||||
fmt = IMPL_FMT(impl, i);
|
||||
cnt += sprintf(buffer + cnt, fmt,
|
||||
generic_supp_impls[i]->name);
|
||||
}
|
||||
|
||||
return (cnt);
|
||||
}
|
||||
|
||||
static int
|
||||
sha512_param_set(const char *val, zfs_kernel_param_t *unused)
|
||||
{
|
||||
(void) unused;
|
||||
return (generic_impl_setname(val));
|
||||
}
|
||||
|
||||
#elif defined(__FreeBSD__)
|
||||
|
||||
#include <sys/sbuf.h>
|
||||
|
||||
static int
|
||||
sha512_param(ZFS_MODULE_PARAM_ARGS)
|
||||
{
|
||||
int err;
|
||||
|
||||
generic_impl_init();
|
||||
if (req->newptr == NULL) {
|
||||
const uint32_t impl = IMPL_READ(generic_impl_chosen);
|
||||
const int init_buflen = 64;
|
||||
const char *fmt;
|
||||
struct sbuf *s;
|
||||
|
||||
s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
|
||||
|
||||
/* cycling */
|
||||
fmt = IMPL_FMT(impl, IMPL_CYCLE);
|
||||
(void) sbuf_printf(s, fmt, "cycle");
|
||||
|
||||
/* list fastest */
|
||||
fmt = IMPL_FMT(impl, IMPL_FASTEST);
|
||||
(void) sbuf_printf(s, fmt, "fastest");
|
||||
|
||||
/* list all supported implementations */
|
||||
for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
|
||||
fmt = IMPL_FMT(impl, i);
|
||||
(void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
|
||||
}
|
||||
|
||||
err = sbuf_finish(s);
|
||||
sbuf_delete(s);
|
||||
|
||||
return (err);
|
||||
}
|
||||
|
||||
/* we got module parameter */
|
||||
char buf[16];
|
||||
|
||||
err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
|
||||
if (err) {
|
||||
return (err);
|
||||
}
|
||||
|
||||
return (-generic_impl_setname(buf));
|
||||
}
|
||||
#endif
|
||||
|
||||
#undef IMPL_FMT
|
||||
|
||||
ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha512_impl,
|
||||
sha512_param_set, sha512_param_get, ZMOD_RW, \
|
||||
"Select SHA512 implementation.");
|
||||
#endif
|
||||
|
||||
#undef TF
|
1999
module/icp/asm-aarch64/sha2/sha256-armv8.S
Normal file
1999
module/icp/asm-aarch64/sha2/sha256-armv8.S
Normal file
File diff suppressed because it is too large
Load Diff
1558
module/icp/asm-aarch64/sha2/sha512-armv8.S
Normal file
1558
module/icp/asm-aarch64/sha2/sha512-armv8.S
Normal file
File diff suppressed because it is too large
Load Diff
2769
module/icp/asm-arm/sha2/sha256-armv7.S
Normal file
2769
module/icp/asm-arm/sha2/sha256-armv7.S
Normal file
File diff suppressed because it is too large
Load Diff
1822
module/icp/asm-arm/sha2/sha512-armv7.S
Normal file
1822
module/icp/asm-arm/sha2/sha512-armv7.S
Normal file
File diff suppressed because it is too large
Load Diff
1505
module/icp/asm-ppc64/sha2/sha256-p8.S
Normal file
1505
module/icp/asm-ppc64/sha2/sha256-p8.S
Normal file
File diff suppressed because it is too large
Load Diff
2712
module/icp/asm-ppc64/sha2/sha256-ppc.S
Normal file
2712
module/icp/asm-ppc64/sha2/sha256-ppc.S
Normal file
File diff suppressed because it is too large
Load Diff
1706
module/icp/asm-ppc64/sha2/sha512-p8.S
Normal file
1706
module/icp/asm-ppc64/sha2/sha512-p8.S
Normal file
File diff suppressed because it is too large
Load Diff
2958
module/icp/asm-ppc64/sha2/sha512-ppc.S
Normal file
2958
module/icp/asm-ppc64/sha2/sha512-ppc.S
Normal file
File diff suppressed because it is too large
Load Diff
5104
module/icp/asm-x86_64/sha2/sha256-x86_64.S
Normal file
5104
module/icp/asm-x86_64/sha2/sha256-x86_64.S
Normal file
File diff suppressed because it is too large
Load Diff
4011
module/icp/asm-x86_64/sha2/sha512-x86_64.S
Normal file
4011
module/icp/asm-x86_64/sha2/sha512-x86_64.S
Normal file
File diff suppressed because it is too large
Load Diff
233
module/icp/include/generic_impl.c
Normal file
233
module/icp/include/generic_impl.c
Normal file
@ -0,0 +1,233 @@
|
||||
/*
|
||||
* CDDL HEADER START
|
||||
*
|
||||
* The contents of this file are subject to the terms of the
|
||||
* Common Development and Distribution License (the "License").
|
||||
* You may not use this file except in compliance with the License.
|
||||
*
|
||||
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
||||
* or https://opensource.org/licenses/CDDL-1.0.
|
||||
* See the License for the specific language governing permissions
|
||||
* and limitations under the License.
|
||||
*
|
||||
* When distributing Covered Code, include this CDDL HEADER in each
|
||||
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
||||
* If applicable, add the following below this CDDL HEADER, with the
|
||||
* fields enclosed by brackets "[]" replaced with your own identifying
|
||||
* information: Portions Copyright [yyyy] [name of copyright owner]
|
||||
*
|
||||
* CDDL HEADER END
|
||||
*/
|
||||
|
||||
/*
|
||||
* Copyright (c) 2003, 2010 Oracle and/or its affiliates.
|
||||
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
|
||||
*/
|
||||
|
||||
/*
|
||||
* This file gets included by c files for implementing the full set
|
||||
* of zfs_impl.h defines.
|
||||
*
|
||||
* It's ment for easier maintaining multiple implementations of
|
||||
* algorithms. Look into blake3_impl.c, sha256_impl.c or sha512_impl.c
|
||||
* for reference.
|
||||
*/
|
||||
|
||||
#include <sys/zfs_context.h>
|
||||
#include <sys/zio_checksum.h>
|
||||
#include <sys/zfs_impl.h>
|
||||
|
||||
/* Two default implementations */
|
||||
#define IMPL_FASTEST (UINT32_MAX)
|
||||
#define IMPL_CYCLE (UINT32_MAX - 1)
|
||||
|
||||
#define IMPL_READ(i) (*(volatile uint32_t *) &(i))
|
||||
|
||||
/* Implementation that contains the fastest method */
|
||||
static IMPL_OPS_T generic_fastest_impl = {
|
||||
.name = "fastest"
|
||||
};
|
||||
|
||||
/* Hold all supported implementations */
|
||||
static const IMPL_OPS_T *generic_supp_impls[ARRAY_SIZE(IMPL_ARRAY)];
|
||||
static uint32_t generic_supp_impls_cnt = 0;
|
||||
|
||||
/* Currently selected implementation */
|
||||
static uint32_t generic_impl_chosen = IMPL_FASTEST;
|
||||
|
||||
static struct generic_impl_selector {
|
||||
const char *name;
|
||||
uint32_t sel;
|
||||
} generic_impl_selectors[] = {
|
||||
{ "cycle", IMPL_CYCLE },
|
||||
{ "fastest", IMPL_FASTEST }
|
||||
};
|
||||
|
||||
/* check the supported implementations */
|
||||
static void
|
||||
generic_impl_init(void)
|
||||
{
|
||||
int i, c;
|
||||
|
||||
/* init only once */
|
||||
if (likely(generic_supp_impls_cnt != 0))
|
||||
return;
|
||||
|
||||
/* Move supported implementations into generic_supp_impls */
|
||||
for (i = 0, c = 0; i < ARRAY_SIZE(IMPL_ARRAY); i++) {
|
||||
const IMPL_OPS_T *impl = IMPL_ARRAY[i];
|
||||
|
||||
if (impl->is_supported && impl->is_supported())
|
||||
generic_supp_impls[c++] = impl;
|
||||
}
|
||||
generic_supp_impls_cnt = c;
|
||||
|
||||
/* first init generic impl, may be changed via set_fastest() */
|
||||
memcpy(&generic_fastest_impl, generic_supp_impls[0],
|
||||
sizeof (generic_fastest_impl));
|
||||
}
|
||||
|
||||
/* get number of supported implementations */
|
||||
static uint32_t
|
||||
generic_impl_getcnt(void)
|
||||
{
|
||||
generic_impl_init();
|
||||
return (generic_supp_impls_cnt);
|
||||
}
|
||||
|
||||
/* get id of selected implementation */
|
||||
static uint32_t
|
||||
generic_impl_getid(void)
|
||||
{
|
||||
generic_impl_init();
|
||||
return (IMPL_READ(generic_impl_chosen));
|
||||
}
|
||||
|
||||
/* get name of selected implementation */
|
||||
static const char *
|
||||
generic_impl_getname(void)
|
||||
{
|
||||
uint32_t impl = IMPL_READ(generic_impl_chosen);
|
||||
|
||||
generic_impl_init();
|
||||
switch (impl) {
|
||||
case IMPL_FASTEST:
|
||||
return ("fastest");
|
||||
case IMPL_CYCLE:
|
||||
return ("cycle");
|
||||
default:
|
||||
return (generic_supp_impls[impl]->name);
|
||||
}
|
||||
}
|
||||
|
||||
/* set implementation by id */
|
||||
static void
|
||||
generic_impl_setid(uint32_t id)
|
||||
{
|
||||
generic_impl_init();
|
||||
switch (id) {
|
||||
case IMPL_FASTEST:
|
||||
atomic_swap_32(&generic_impl_chosen, IMPL_FASTEST);
|
||||
break;
|
||||
case IMPL_CYCLE:
|
||||
atomic_swap_32(&generic_impl_chosen, IMPL_CYCLE);
|
||||
break;
|
||||
default:
|
||||
ASSERT3U(id, <, generic_supp_impls_cnt);
|
||||
atomic_swap_32(&generic_impl_chosen, id);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* set implementation by name */
|
||||
static int
|
||||
generic_impl_setname(const char *val)
|
||||
{
|
||||
uint32_t impl = IMPL_READ(generic_impl_chosen);
|
||||
size_t val_len;
|
||||
int i, err = -EINVAL;
|
||||
|
||||
generic_impl_init();
|
||||
val_len = strlen(val);
|
||||
while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */
|
||||
val_len--;
|
||||
|
||||
/* check mandatory implementations */
|
||||
for (i = 0; i < ARRAY_SIZE(generic_impl_selectors); i++) {
|
||||
const char *name = generic_impl_selectors[i].name;
|
||||
|
||||
if (val_len == strlen(name) &&
|
||||
strncmp(val, name, val_len) == 0) {
|
||||
impl = generic_impl_selectors[i].sel;
|
||||
err = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* check all supported implementations */
|
||||
if (err != 0) {
|
||||
for (i = 0; i < generic_supp_impls_cnt; i++) {
|
||||
const char *name = generic_supp_impls[i]->name;
|
||||
|
||||
if (val_len == strlen(name) &&
|
||||
strncmp(val, name, val_len) == 0) {
|
||||
impl = i;
|
||||
err = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (err == 0) {
|
||||
atomic_swap_32(&generic_impl_chosen, impl);
|
||||
}
|
||||
|
||||
return (err);
|
||||
}
|
||||
|
||||
/* setup id as fastest implementation */
|
||||
static void
|
||||
generic_impl_set_fastest(uint32_t id)
|
||||
{
|
||||
generic_impl_init();
|
||||
memcpy(&generic_fastest_impl, generic_supp_impls[id],
|
||||
sizeof (generic_fastest_impl));
|
||||
}
|
||||
|
||||
/* return impl iterating functions */
|
||||
const zfs_impl_t ZFS_IMPL_OPS = {
|
||||
.name = IMPL_NAME,
|
||||
.getcnt = generic_impl_getcnt,
|
||||
.getid = generic_impl_getid,
|
||||
.getname = generic_impl_getname,
|
||||
.set_fastest = generic_impl_set_fastest,
|
||||
.setid = generic_impl_setid,
|
||||
.setname = generic_impl_setname
|
||||
};
|
||||
|
||||
/* get impl ops_t of selected implementation */
|
||||
const IMPL_OPS_T *
|
||||
IMPL_GET_OPS(void)
|
||||
{
|
||||
const IMPL_OPS_T *ops = NULL;
|
||||
uint32_t idx, impl = IMPL_READ(generic_impl_chosen);
|
||||
static uint32_t cycle_count = 0;
|
||||
|
||||
generic_impl_init();
|
||||
switch (impl) {
|
||||
case IMPL_FASTEST:
|
||||
ops = &generic_fastest_impl;
|
||||
break;
|
||||
case IMPL_CYCLE:
|
||||
idx = (++cycle_count) % generic_supp_impls_cnt;
|
||||
ops = generic_supp_impls[idx];
|
||||
break;
|
||||
default:
|
||||
ASSERT3U(impl, <, generic_supp_impls_cnt);
|
||||
ops = generic_supp_impls[impl];
|
||||
break;
|
||||
}
|
||||
|
||||
ASSERT3P(ops, !=, NULL);
|
||||
return (ops);
|
||||
}
|
@ -18,9 +18,10 @@
|
||||
*
|
||||
* CDDL HEADER END
|
||||
*/
|
||||
|
||||
/*
|
||||
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
|
||||
* Use is subject to license terms.
|
||||
* Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved.
|
||||
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
|
||||
*/
|
||||
|
||||
#ifndef _SHA2_IMPL_H
|
||||
@ -32,6 +33,28 @@
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* transform function definition */
|
||||
typedef void (*sha256_f)(uint32_t state[8], const void *data, size_t blks);
|
||||
typedef void (*sha512_f)(uint64_t state[8], const void *data, size_t blks);
|
||||
|
||||
/* needed for checking valid implementations */
|
||||
typedef boolean_t (*sha2_is_supported_f)(void);
|
||||
|
||||
typedef struct {
|
||||
const char *name;
|
||||
sha256_f transform;
|
||||
sha2_is_supported_f is_supported;
|
||||
} sha256_ops_t;
|
||||
|
||||
typedef struct {
|
||||
const char *name;
|
||||
sha512_f transform;
|
||||
sha2_is_supported_f is_supported;
|
||||
} sha512_ops_t;
|
||||
|
||||
extern const sha256_ops_t *sha256_get_ops(void);
|
||||
extern const sha512_ops_t *sha512_get_ops(void);
|
||||
|
||||
typedef enum {
|
||||
SHA1_TYPE,
|
||||
SHA256_TYPE,
|
||||
|
@ -28,7 +28,6 @@
|
||||
#include <sys/crypto/common.h>
|
||||
#include <sys/crypto/spi.h>
|
||||
#include <sys/crypto/icp.h>
|
||||
#define _SHA2_IMPL
|
||||
#include <sys/sha2.h>
|
||||
#include <sha2/sha2_impl.h>
|
||||
|
||||
|
@ -18,16 +18,14 @@
|
||||
*
|
||||
* CDDL HEADER END
|
||||
*/
|
||||
|
||||
/*
|
||||
* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
|
||||
* Use is subject to license terms.
|
||||
*/
|
||||
/*
|
||||
* Copyright 2013 Saso Kiselkov. All rights reserved.
|
||||
* Copyright (c) 2016 by Delphix. All rights reserved.
|
||||
*/
|
||||
|
||||
#include <sys/zfs_context.h>
|
||||
#include <sys/zio.h>
|
||||
#include <sys/zio_checksum.h>
|
||||
#include <sys/sha2.h>
|
||||
#include <sys/abd.h>
|
||||
@ -42,7 +40,7 @@ sha_incremental(void *buf, size_t size, void *arg)
|
||||
}
|
||||
|
||||
void
|
||||
abd_checksum_SHA256(abd_t *abd, uint64_t size,
|
||||
abd_checksum_sha256(abd_t *abd, uint64_t size,
|
||||
const void *ctx_template, zio_cksum_t *zcp)
|
||||
{
|
||||
(void) ctx_template;
|
||||
@ -79,7 +77,7 @@ abd_checksum_SHA256(abd_t *abd, uint64_t size,
|
||||
}
|
||||
|
||||
void
|
||||
abd_checksum_SHA512_native(abd_t *abd, uint64_t size,
|
||||
abd_checksum_sha512_native(abd_t *abd, uint64_t size,
|
||||
const void *ctx_template, zio_cksum_t *zcp)
|
||||
{
|
||||
(void) ctx_template;
|
||||
@ -91,12 +89,12 @@ abd_checksum_SHA512_native(abd_t *abd, uint64_t size,
|
||||
}
|
||||
|
||||
void
|
||||
abd_checksum_SHA512_byteswap(abd_t *abd, uint64_t size,
|
||||
abd_checksum_sha512_byteswap(abd_t *abd, uint64_t size,
|
||||
const void *ctx_template, zio_cksum_t *zcp)
|
||||
{
|
||||
zio_cksum_t tmp;
|
||||
|
||||
abd_checksum_SHA512_native(abd, size, ctx_template, &tmp);
|
||||
abd_checksum_sha512_native(abd, size, ctx_template, &tmp);
|
||||
zcp->zc_word[0] = BSWAP_64(tmp.zc_word[0]);
|
||||
zcp->zc_word[1] = BSWAP_64(tmp.zc_word[1]);
|
||||
zcp->zc_word[2] = BSWAP_64(tmp.zc_word[2]);
|
@ -23,13 +23,13 @@
|
||||
* Copyright (c) 2021-2022 Tino Reichardt <milky-zfs@mcmilk.de>
|
||||
*/
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/spa.h>
|
||||
#include <sys/zio_checksum.h>
|
||||
#include <sys/zfs_context.h>
|
||||
#include <sys/zfs_chksum.h>
|
||||
#include <sys/zfs_impl.h>
|
||||
|
||||
#include <sys/blake3.h>
|
||||
#include <sys/sha2.h>
|
||||
|
||||
/* limit benchmarking to max 256KiB, when EdonR is slower then this: */
|
||||
#define LIMIT_PERF_MBS 300
|
||||
@ -56,25 +56,26 @@ static int chksum_stat_cnt = 0;
|
||||
static kstat_t *chksum_kstat = NULL;
|
||||
|
||||
/*
|
||||
* i3-1005G1 test output:
|
||||
* Sample output on i3-1005G1 System:
|
||||
*
|
||||
* implementation 1k 4k 16k 64k 256k 1m 4m
|
||||
* fletcher-4 5421 15001 26468 32555 34720 32801 18847
|
||||
* edonr-generic 1196 1602 1761 1749 1762 1759 1751
|
||||
* skein-generic 546 591 608 615 619 612 616
|
||||
* sha256-generic 246 270 274 274 277 275 276
|
||||
* sha256-avx 262 296 304 307 307 307 306
|
||||
* sha256-sha-ni 769 1072 1172 1220 1219 1232 1228
|
||||
* sha256-openssl 240 300 316 314 304 285 276
|
||||
* sha512-generic 333 374 385 392 391 393 392
|
||||
* sha512-openssl 353 441 467 476 472 467 426
|
||||
* sha512-avx 362 444 473 475 479 476 478
|
||||
* sha512-avx2 394 500 530 538 543 545 542
|
||||
* blake3-generic 308 313 313 313 312 313 312
|
||||
* blake3-sse2 402 1289 1423 1446 1432 1458 1413
|
||||
* blake3-sse41 427 1470 1625 1704 1679 1607 1629
|
||||
* blake3-avx2 428 1920 3095 3343 3356 3318 3204
|
||||
* blake3-avx512 473 2687 4905 5836 5844 5643 5374
|
||||
* implementation 1k 4k 16k 64k 256k 1m 4m 16m
|
||||
* edonr-generic 1278 1625 1769 1776 1783 1778 1771 1767
|
||||
* skein-generic 548 594 613 623 621 623 621 486
|
||||
* sha256-generic 255 270 281 278 279 281 283 283
|
||||
* sha256-x64 288 310 316 317 318 317 317 316
|
||||
* sha256-ssse3 304 342 351 355 356 357 356 356
|
||||
* sha256-avx 311 348 359 362 362 363 363 362
|
||||
* sha256-avx2 330 378 389 395 395 395 395 395
|
||||
* sha256-shani 908 1127 1212 1230 1233 1234 1223 1230
|
||||
* sha512-generic 359 409 431 427 429 430 428 423
|
||||
* sha512-x64 420 473 490 496 497 497 496 495
|
||||
* sha512-avx 406 522 546 560 560 560 556 560
|
||||
* sha512-avx2 464 568 601 606 609 610 607 608
|
||||
* blake3-generic 330 327 324 323 324 320 323 322
|
||||
* blake3-sse2 424 1366 1449 1468 1458 1453 1395 1408
|
||||
* blake3-sse41 453 1554 1658 1703 1689 1669 1622 1630
|
||||
* blake3-avx2 452 2013 3225 3351 3356 3261 3076 3101
|
||||
* blake3-avx512 498 2869 5269 5926 5872 5643 5014 5005
|
||||
*/
|
||||
static int
|
||||
chksum_kstat_headers(char *buf, size_t size)
|
||||
@ -237,25 +238,30 @@ chksum_benchit(chksum_stat_t *cs)
|
||||
static void
|
||||
chksum_benchmark(void)
|
||||
{
|
||||
|
||||
#ifndef _KERNEL
|
||||
/* we need the benchmark only for the kernel module */
|
||||
return;
|
||||
#endif
|
||||
|
||||
chksum_stat_t *cs;
|
||||
int cbid = 0;
|
||||
uint64_t max = 0;
|
||||
uint32_t id, id_save;
|
||||
uint64_t max;
|
||||
uint32_t id, cbid = 0, id_save;
|
||||
const zfs_impl_t *blake3 = zfs_impl_get_ops("blake3");
|
||||
const zfs_impl_t *sha256 = zfs_impl_get_ops("sha256");
|
||||
const zfs_impl_t *sha512 = zfs_impl_get_ops("sha512");
|
||||
|
||||
/* space for the benchmark times */
|
||||
chksum_stat_cnt = 4;
|
||||
chksum_stat_cnt += blake3_impl_getcnt();
|
||||
/* count implementations */
|
||||
chksum_stat_cnt = 2;
|
||||
chksum_stat_cnt += sha256->getcnt();
|
||||
chksum_stat_cnt += sha512->getcnt();
|
||||
chksum_stat_cnt += blake3->getcnt();
|
||||
chksum_stat_data = kmem_zalloc(
|
||||
sizeof (chksum_stat_t) * chksum_stat_cnt, KM_SLEEP);
|
||||
|
||||
/* edonr - needs to be the first one here (slow CPU check) */
|
||||
cs = &chksum_stat_data[cbid++];
|
||||
|
||||
/* edonr */
|
||||
cs->init = abd_checksum_edonr_tmpl_init;
|
||||
cs->func = abd_checksum_edonr_native;
|
||||
cs->free = abd_checksum_edonr_tmpl_free;
|
||||
@ -273,42 +279,58 @@ chksum_benchmark(void)
|
||||
chksum_benchit(cs);
|
||||
|
||||
/* sha256 */
|
||||
cs = &chksum_stat_data[cbid++];
|
||||
cs->init = 0;
|
||||
cs->func = abd_checksum_SHA256;
|
||||
cs->free = 0;
|
||||
cs->name = "sha256";
|
||||
cs->impl = "generic";
|
||||
chksum_benchit(cs);
|
||||
id_save = sha256->getid();
|
||||
for (max = 0, id = 0; id < sha256->getcnt(); id++) {
|
||||
sha256->setid(id);
|
||||
cs = &chksum_stat_data[cbid++];
|
||||
cs->init = 0;
|
||||
cs->func = abd_checksum_sha256;
|
||||
cs->free = 0;
|
||||
cs->name = sha256->name;
|
||||
cs->impl = sha256->getname();
|
||||
chksum_benchit(cs);
|
||||
if (cs->bs256k > max) {
|
||||
max = cs->bs256k;
|
||||
sha256->set_fastest(id);
|
||||
}
|
||||
}
|
||||
sha256->setid(id_save);
|
||||
|
||||
/* sha512 */
|
||||
cs = &chksum_stat_data[cbid++];
|
||||
cs->init = 0;
|
||||
cs->func = abd_checksum_SHA512_native;
|
||||
cs->free = 0;
|
||||
cs->name = "sha512";
|
||||
cs->impl = "generic";
|
||||
chksum_benchit(cs);
|
||||
id_save = sha512->getid();
|
||||
for (max = 0, id = 0; id < sha512->getcnt(); id++) {
|
||||
sha512->setid(id);
|
||||
cs = &chksum_stat_data[cbid++];
|
||||
cs->init = 0;
|
||||
cs->func = abd_checksum_sha512_native;
|
||||
cs->free = 0;
|
||||
cs->name = sha512->name;
|
||||
cs->impl = sha512->getname();
|
||||
chksum_benchit(cs);
|
||||
if (cs->bs256k > max) {
|
||||
max = cs->bs256k;
|
||||
sha512->set_fastest(id);
|
||||
}
|
||||
}
|
||||
sha512->setid(id_save);
|
||||
|
||||
/* blake3 */
|
||||
id_save = blake3_impl_getid();
|
||||
for (id = 0; id < blake3_impl_getcnt(); id++) {
|
||||
blake3_impl_setid(id);
|
||||
id_save = blake3->getid();
|
||||
for (max = 0, id = 0; id < blake3->getcnt(); id++) {
|
||||
blake3->setid(id);
|
||||
cs = &chksum_stat_data[cbid++];
|
||||
cs->init = abd_checksum_blake3_tmpl_init;
|
||||
cs->func = abd_checksum_blake3_native;
|
||||
cs->free = abd_checksum_blake3_tmpl_free;
|
||||
cs->name = "blake3";
|
||||
cs->impl = blake3_impl_getname();
|
||||
cs->name = blake3->name;
|
||||
cs->impl = blake3->getname();
|
||||
chksum_benchit(cs);
|
||||
if (cs->bs256k > max) {
|
||||
max = cs->bs256k;
|
||||
blake3_impl_set_fastest(id);
|
||||
blake3->set_fastest(id);
|
||||
}
|
||||
}
|
||||
|
||||
/* restore initial value */
|
||||
blake3_impl_setid(id_save);
|
||||
blake3->setid(id_save);
|
||||
}
|
||||
|
||||
void
|
||||
|
61
module/zfs/zfs_impl.c
Normal file
61
module/zfs/zfs_impl.c
Normal file
@ -0,0 +1,61 @@
|
||||
/*
|
||||
* CDDL HEADER START
|
||||
*
|
||||
* The contents of this file are subject to the terms of the
|
||||
* Common Development and Distribution License (the "License").
|
||||
* You may not use this file except in compliance with the License.
|
||||
*
|
||||
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
|
||||
* or https://opensource.org/licenses/CDDL-1.0.
|
||||
* See the License for the specific language governing permissions
|
||||
* and limitations under the License.
|
||||
*
|
||||
* When distributing Covered Code, include this CDDL HEADER in each
|
||||
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
|
||||
* If applicable, add the following below this CDDL HEADER, with the
|
||||
* fields enclosed by brackets "[]" replaced with your own identifying
|
||||
* information: Portions Copyright [yyyy] [name of copyright owner]
|
||||
*
|
||||
* CDDL HEADER END
|
||||
*/
|
||||
|
||||
/*
|
||||
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
|
||||
*/
|
||||
|
||||
#include <sys/zio_checksum.h>
|
||||
#include <sys/zfs_context.h>
|
||||
#include <sys/zfs_impl.h>
|
||||
|
||||
#include <sys/blake3.h>
|
||||
#include <sys/sha2.h>
|
||||
|
||||
/*
|
||||
* impl_ops - backend for implementations of algorithms
|
||||
*/
|
||||
const zfs_impl_t *impl_ops[] = {
|
||||
&zfs_blake3_ops,
|
||||
&zfs_sha256_ops,
|
||||
&zfs_sha512_ops,
|
||||
NULL
|
||||
};
|
||||
|
||||
/*
|
||||
* zfs_impl_get_ops - Get the API functions for an impl backend
|
||||
*/
|
||||
const zfs_impl_t *
|
||||
zfs_impl_get_ops(const char *algo)
|
||||
{
|
||||
const zfs_impl_t **ops = impl_ops;
|
||||
|
||||
if (!algo || !*algo)
|
||||
return (*ops);
|
||||
|
||||
for (; *ops; ops++) {
|
||||
if (strcmp(algo, (*ops)->name) == 0)
|
||||
break;
|
||||
}
|
||||
|
||||
ASSERT3P(ops, !=, NULL);
|
||||
return (*ops);
|
||||
}
|
@ -165,10 +165,10 @@ zio_checksum_info_t zio_checksum_table[ZIO_CHECKSUM_FUNCTIONS] = {
|
||||
{{NULL, NULL}, NULL, NULL, 0, "on"},
|
||||
{{abd_checksum_off, abd_checksum_off},
|
||||
NULL, NULL, 0, "off"},
|
||||
{{abd_checksum_SHA256, abd_checksum_SHA256},
|
||||
{{abd_checksum_sha256, abd_checksum_sha256},
|
||||
NULL, NULL, ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_EMBEDDED,
|
||||
"label"},
|
||||
{{abd_checksum_SHA256, abd_checksum_SHA256},
|
||||
{{abd_checksum_sha256, abd_checksum_sha256},
|
||||
NULL, NULL, ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_EMBEDDED,
|
||||
"gang_header"},
|
||||
{{abd_fletcher_2_native, abd_fletcher_2_byteswap},
|
||||
@ -177,14 +177,14 @@ zio_checksum_info_t zio_checksum_table[ZIO_CHECKSUM_FUNCTIONS] = {
|
||||
NULL, NULL, 0, "fletcher2"},
|
||||
{{abd_fletcher_4_native, abd_fletcher_4_byteswap},
|
||||
NULL, NULL, ZCHECKSUM_FLAG_METADATA, "fletcher4"},
|
||||
{{abd_checksum_SHA256, abd_checksum_SHA256},
|
||||
{{abd_checksum_sha256, abd_checksum_sha256},
|
||||
NULL, NULL, ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_DEDUP |
|
||||
ZCHECKSUM_FLAG_NOPWRITE, "sha256"},
|
||||
{{abd_fletcher_4_native, abd_fletcher_4_byteswap},
|
||||
NULL, NULL, ZCHECKSUM_FLAG_EMBEDDED, "zilog2"},
|
||||
{{abd_checksum_off, abd_checksum_off},
|
||||
NULL, NULL, 0, "noparity"},
|
||||
{{abd_checksum_SHA512_native, abd_checksum_SHA512_byteswap},
|
||||
{{abd_checksum_sha512_native, abd_checksum_sha512_byteswap},
|
||||
NULL, NULL, ZCHECKSUM_FLAG_METADATA | ZCHECKSUM_FLAG_DEDUP |
|
||||
ZCHECKSUM_FLAG_NOPWRITE, "sha512"},
|
||||
{{abd_checksum_skein_native, abd_checksum_skein_byteswap},
|
||||
|
@ -33,11 +33,11 @@
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
|
||||
#include <sys/time.h>
|
||||
#define _SHA2_IMPL
|
||||
#include <sys/sha2.h>
|
||||
#include <sys/stdtypes.h>
|
||||
|
||||
#include <sys/zfs_impl.h>
|
||||
|
||||
/*
|
||||
* Test messages from:
|
||||
@ -174,9 +174,19 @@ main(int argc, char *argv[])
|
||||
boolean_t failed = B_FALSE;
|
||||
uint64_t cpu_mhz = 0;
|
||||
|
||||
const zfs_impl_t *sha256 = zfs_impl_get_ops("sha256");
|
||||
const zfs_impl_t *sha512 = zfs_impl_get_ops("sha512");
|
||||
uint32_t id;
|
||||
|
||||
if (argc == 2)
|
||||
cpu_mhz = atoi(argv[1]);
|
||||
|
||||
if (!sha256)
|
||||
return (1);
|
||||
|
||||
if (!sha512)
|
||||
return (1);
|
||||
|
||||
#define SHA2_ALGO_TEST(_m, mode, diglen, testdigest) \
|
||||
do { \
|
||||
SHA2_CTX ctx; \
|
||||
@ -194,7 +204,7 @@ main(int argc, char *argv[])
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define SHA2_PERF_TEST(mode, diglen) \
|
||||
#define SHA2_PERF_TEST(mode, diglen, name) \
|
||||
do { \
|
||||
SHA2_CTX ctx; \
|
||||
uint8_t digest[diglen / 8]; \
|
||||
@ -216,8 +226,8 @@ main(int argc, char *argv[])
|
||||
cpb = (cpu_mhz * 1e6 * ((double)delta / \
|
||||
1000000)) / (8192 * 128 * 1024); \
|
||||
} \
|
||||
(void) printf("SHA%-9s%llu us (%.02f CPB)\n", #mode, \
|
||||
(u_longlong_t)delta, cpb); \
|
||||
(void) printf("sha%s-%-9s%7llu us (%.02f CPB)\n", #mode,\
|
||||
name, (u_longlong_t)delta, cpb); \
|
||||
} while (0)
|
||||
|
||||
(void) printf("Running algorithm correctness tests:\n");
|
||||
@ -237,8 +247,18 @@ main(int argc, char *argv[])
|
||||
|
||||
(void) printf("Running performance tests (hashing 1024 MiB of "
|
||||
"data):\n");
|
||||
SHA2_PERF_TEST(256, 256);
|
||||
SHA2_PERF_TEST(512, 512);
|
||||
|
||||
for (id = 0; id < sha256->getcnt(); id++) {
|
||||
sha256->setid(id);
|
||||
const char *name = sha256->getname();
|
||||
SHA2_PERF_TEST(256, 256, name);
|
||||
}
|
||||
|
||||
for (id = 0; id < sha512->getcnt(); id++) {
|
||||
sha512->setid(id);
|
||||
const char *name = sha512->getname();
|
||||
SHA2_PERF_TEST(512, 512, name);
|
||||
}
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user