d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Restore ASMABI and other Unify work

Browse Source 
Make sure all SHA2 transform function has wrappers

For ASMABI to work, it is required the calling convention
is consistent.

Reviewed-by: Tino Reichardt <milky-zfs@mcmilk.de>
Reviewed-by: Richard Yao <richard.yao@alumni.stonybrook.edu>
Signed-off-by: Joergen Lundman <lundman@lundman.net>
Closes #14569
This commit is contained in:
Jorgen Lundman 2023-03-07 08:24:05 +09:00 committed by GitHub
parent 620a977f22
commit 47119d60ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 56 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
module/icp/algs/blake3/blake3_impl.c
View File

@ -34,15 +34,15 @@
(defined(__x86_64) && defined(HAVE_SSE2)) || \
(defined(__PPC64__) && defined(__LITTLE_ENDIAN__))
extern void zfs_blake3_compress_in_place_sse2(uint32_t cv[8],
extern void ASMABI zfs_blake3_compress_in_place_sse2(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
uint64_t counter, uint8_t flags);
extern void zfs_blake3_compress_xof_sse2(const uint32_t cv[8],
extern void ASMABI zfs_blake3_compress_xof_sse2(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
uint64_t counter, uint8_t flags, uint8_t out[64]);
extern void zfs_blake3_hash_many_sse2(const uint8_t * const *inputs,
extern void ASMABI zfs_blake3_hash_many_sse2(const uint8_t * const *inputs,
size_t num_inputs, size_t blocks, const uint32_t key[8],
uint64_t counter, boolean_t increment_counter, uint8_t flags,
uint8_t flags_start, uint8_t flags_end, uint8_t *out);
@ -100,15 +100,15 @@ const blake3_ops_t blake3_sse2_impl = {
(defined(__x86_64) && defined(HAVE_SSE2)) || \
(defined(__PPC64__) && defined(__LITTLE_ENDIAN__))
extern void zfs_blake3_compress_in_place_sse41(uint32_t cv[8],
extern void ASMABI zfs_blake3_compress_in_place_sse41(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
uint64_t counter, uint8_t flags);
extern void zfs_blake3_compress_xof_sse41(const uint32_t cv[8],
extern void ASMABI zfs_blake3_compress_xof_sse41(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
uint64_t counter, uint8_t flags, uint8_t out[64]);
extern void zfs_blake3_hash_many_sse41(const uint8_t * const *inputs,
extern void ASMABI zfs_blake3_hash_many_sse41(const uint8_t * const *inputs,
size_t num_inputs, size_t blocks, const uint32_t key[8],
uint64_t counter, boolean_t increment_counter, uint8_t flags,
uint8_t flags_start, uint8_t flags_end, uint8_t *out);
@ -163,7 +163,7 @@ const blake3_ops_t blake3_sse41_impl = {
#endif
#if defined(__x86_64) && defined(HAVE_SSE4_1) && defined(HAVE_AVX2)
extern void zfs_blake3_hash_many_avx2(const uint8_t * const *inputs,
extern void ASMABI zfs_blake3_hash_many_avx2(const uint8_t * const *inputs,
size_t num_inputs, size_t blocks, const uint32_t key[8],
uint64_t counter, boolean_t increment_counter, uint8_t flags,
uint8_t flags_start, uint8_t flags_end, uint8_t *out);
@ -196,15 +196,15 @@ blake3_avx2_impl = {
#endif
#if defined(__x86_64) && defined(HAVE_AVX512F) && defined(HAVE_AVX512VL)
extern void zfs_blake3_compress_in_place_avx512(uint32_t cv[8],
extern void ASMABI zfs_blake3_compress_in_place_avx512(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
uint64_t counter, uint8_t flags);
extern void zfs_blake3_compress_xof_avx512(const uint32_t cv[8],
extern void ASMABI zfs_blake3_compress_xof_avx512(const uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
uint64_t counter, uint8_t flags, uint8_t out[64]);
extern void zfs_blake3_hash_many_avx512(const uint8_t * const *inputs,
extern void ASMABI zfs_blake3_hash_many_avx512(const uint8_t * const *inputs,
size_t num_inputs, size_t blocks, const uint32_t key[8],
uint64_t counter, boolean_t increment_counter, uint8_t flags,
uint8_t flags_start, uint8_t flags_end, uint8_t *out);

16
module/icp/algs/sha2/sha256_impl.c
View File

@ -29,9 +29,10 @@
#include <sys/simd.h>
#include <sha2/sha2_impl.h>
#include <sys/asm_linkage.h>
#define TF(E, N) \
extern void E(uint32_t s[8], const void *, size_t); \
extern void ASMABI E(uint32_t s[8], const void *, size_t); \
static inline void N(uint32_t s[8], const void *d, size_t b) { \
kfpu_begin(); E(s, d, b); kfpu_end(); \
}
@ -44,10 +45,19 @@ static inline boolean_t sha2_is_supported(void)
#if defined(__x86_64)
extern void zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t);
/* Users of ASMABI requires all calls to be from wrappers */
extern void ASMABI
zfs_sha256_transform_x64(uint32_t s[8], const void *, size_t);
static inline void
tf_sha256_transform_x64(uint32_t s[8], const void *d, size_t b)
{
zfs_sha256_transform_x64(s, d, b);
}
const sha256_ops_t sha256_x64_impl = {
.is_supported = sha2_is_supported,
.transform = zfs_sha256_transform_x64,
.transform = tf_sha256_transform_x64,
.name = "x64"
};

15
module/icp/algs/sha2/sha512_impl.c
View File

@ -29,9 +29,10 @@
#include <sys/simd.h>
#include <sha2/sha2_impl.h>
#include <sys/asm_linkage.h>
#define TF(E, N) \
extern void E(uint64_t s[8], const void *, size_t); \
extern void ASMABI E(uint64_t s[8], const void *, size_t); \
static inline void N(uint64_t s[8], const void *d, size_t b) { \
kfpu_begin(); E(s, d, b); kfpu_end(); \
}
@ -44,10 +45,18 @@ static inline boolean_t sha2_is_supported(void)
#if defined(__x86_64)
extern void zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t);
/* Users of ASMABI requires all calls to be from wrappers */
extern void ASMABI
zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t);
static inline void
tf_sha512_transform_x64(uint64_t s[8], const void *d, size_t b)
{
zfs_sha512_transform_x64(s, d, b);
}
const sha512_ops_t sha512_x64_impl = {
.is_supported = sha2_is_supported,
.transform = zfs_sha512_transform_x64,
.transform = tf_sha512_transform_x64,
.name = "x64"
};

24
module/icp/asm-x86_64/sha2/sha256-x86_64.S
View File

@ -26,8 +26,8 @@
SECTION_STATIC
.align 64
.type K256,@object
.balign 64
SET_OBJ(K256)
K256:
.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
@ -105,7 +105,7 @@ ENTRY_ALIGN(zfs_sha256_transform_x64, 16)
movl 24(%rdi),%r10d
movl 28(%rdi),%r11d
jmp .Lloop
.align 16
.balign 16
.Lloop:
movl %ebx,%edi
leaq K256(%rip),%rbp
@ -622,7 +622,7 @@ ENTRY_ALIGN(zfs_sha256_transform_x64, 16)
addl %r12d,%eax
leaq 20(%rbp),%rbp
jmp .Lrounds_16_xx
.align 16
.balign 16
.Lrounds_16_xx:
movl 4(%rsp),%r13d
movl 56(%rsp),%r15d
@ -1436,7 +1436,7 @@ ENTRY_ALIGN(zfs_sha256_transform_shani, 64)
punpcklqdq %xmm0,%xmm2
jmp .Loop_shani
.align 16
.balign 16
.Loop_shani:
movdqu (%rsi),%xmm3
movdqu 16(%rsi),%xmm4
@ -1666,7 +1666,7 @@ ENTRY_ALIGN(zfs_sha256_transform_ssse3, 64)
movl 28(%rdi),%r11d
jmp .Lloop_ssse3
.align 16
.balign 16
.Lloop_ssse3:
movdqa K256+512(%rip),%xmm7
movdqu 0(%rsi),%xmm0
@ -1696,7 +1696,7 @@ ENTRY_ALIGN(zfs_sha256_transform_ssse3, 64)
movl %r8d,%r13d
jmp .Lssse3_00_47
.align 16
.balign 16
.Lssse3_00_47:
subq $-128,%rbp
rorl $14,%r13d
@ -2779,7 +2779,7 @@ ENTRY_ALIGN(zfs_sha256_transform_avx, 64)
vmovdqa K256+512+32(%rip),%xmm8
vmovdqa K256+512+64(%rip),%xmm9
jmp .Lloop_avx
.align 16
.balign 16
.Lloop_avx:
vmovdqa K256+512(%rip),%xmm7
vmovdqu 0(%rsi),%xmm0
@ -2805,7 +2805,7 @@ ENTRY_ALIGN(zfs_sha256_transform_avx, 64)
movl %r8d,%r13d
jmp .Lavx_00_47
.align 16
.balign 16
.Lavx_00_47:
subq $-128,%rbp
vpalignr $4,%xmm0,%xmm1,%xmm4
@ -3858,7 +3858,7 @@ ENTRY_ALIGN(zfs_sha256_transform_avx2, 64)
vmovdqa K256+512+32(%rip),%ymm8
vmovdqa K256+512+64(%rip),%ymm9
jmp .Loop_avx2
.align 16
.balign 16
.Loop_avx2:
vmovdqa K256+512(%rip),%ymm7
vmovdqu -64+0(%rsi),%xmm0
@ -3900,7 +3900,7 @@ ENTRY_ALIGN(zfs_sha256_transform_avx2, 64)
subq $-32*4,%rbp
jmp .Lavx2_00_47
.align 16
.balign 16
.Lavx2_00_47:
leaq -64(%rsp),%rsp
.cfi_escape 0x0f,0x05,0x77,0x38,0x06,0x23,0x08
@ -4842,7 +4842,7 @@ ENTRY_ALIGN(zfs_sha256_transform_avx2, 64)
xorl %ecx,%edi
movl %r9d,%r12d
jmp .Lower_avx2
.align 16
.balign 16
.Lower_avx2:
addl 0+16(%rbp),%r11d
andl %r8d,%r12d

18
module/icp/asm-x86_64/sha2/sha512-x86_64.S
View File

@ -26,8 +26,8 @@
SECTION_STATIC
.align 64
.type K512,@object
.balign 64
SET_OBJ(K512)
K512:
.quad 0x428a2f98d728ae22,0x7137449123ef65cd
.quad 0x428a2f98d728ae22,0x7137449123ef65cd
@ -148,7 +148,7 @@ ENTRY_ALIGN(zfs_sha512_transform_x64, 16)
movq 48(%rdi),%r10
movq 56(%rdi),%r11
jmp .Lloop
.align 16
.balign 16
.Lloop:
movq %rbx,%rdi
leaq K512(%rip),%rbp
@ -665,7 +665,7 @@ ENTRY_ALIGN(zfs_sha512_transform_x64, 16)
addq %r12,%rax
leaq 24(%rbp),%rbp
jmp .Lrounds_16_xx
.align 16
.balign 16
.Lrounds_16_xx:
movq 8(%rsp),%r13
movq 112(%rsp),%r15
@ -1501,7 +1501,7 @@ ENTRY_ALIGN(zfs_sha512_transform_avx, 64)
movq 48(%rdi),%r10
movq 56(%rdi),%r11
jmp .Lloop_avx
.align 16
.balign 16
.Lloop_avx:
vmovdqa K512+1280(%rip),%xmm11
vmovdqu 0(%rsi),%xmm0
@ -1543,7 +1543,7 @@ ENTRY_ALIGN(zfs_sha512_transform_avx, 64)
movq %r8,%r13
jmp .Lavx_00_47
.align 16
.balign 16
.Lavx_00_47:
addq $256,%rbp
vpalignr $8,%xmm0,%xmm1,%xmm8
@ -2670,7 +2670,7 @@ ENTRY_ALIGN(zfs_sha512_transform_avx2, 64)
movq 48(%rdi),%r10
movq 56(%rdi),%r11
jmp .Loop_avx2
.align 16
.balign 16
.Loop_avx2:
vmovdqu -128(%rsi),%xmm0
vmovdqu -128+16(%rsi),%xmm1
@ -2732,7 +2732,7 @@ ENTRY_ALIGN(zfs_sha512_transform_avx2, 64)
addq $32*8,%rbp
jmp .Lavx2_00_47
.align 16
.balign 16
.Lavx2_00_47:
leaq -128(%rsp),%rsp
.cfi_escape 0x0f,0x06,0x77,0xf8,0x00,0x06,0x23,0x08
@ -3750,7 +3750,7 @@ ENTRY_ALIGN(zfs_sha512_transform_avx2, 64)
xorq %rcx,%rdi
movq %r9,%r12
jmp .Lower_avx2
.align 16
.balign 16
.Lower_avx2:
addq 0+16(%rbp),%r11
andq %r8,%r12