diff --git a/share/man/man4/ossl.4 b/share/man/man4/ossl.4 index 5929e46e9fe3..039ce301ac29 100644 --- a/share/man/man4/ossl.4 +++ b/share/man/man4/ossl.4 @@ -26,7 +26,7 @@ .\" .\" $FreeBSD$ .\" -.Dd March 3, 2021 +.Dd September 24, 2021 .Dt OSSL 4 .Os .Sh NAME @@ -74,6 +74,8 @@ driver includes support for the following algorithms: .Pp .Bl -bullet -compact .It +AES-CBC +.It ChaCha20 .It ChaCha20-Poly1305 (RFC 8439) diff --git a/sys/conf/files b/sys/conf/files index 5fa918b59f62..e1ee0e9fc9ef 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -716,6 +716,7 @@ crypto/chacha20/chacha-sw.c optional crypto | ipsec | ipsec_support crypto/des/des_ecb.c optional netsmb crypto/des/des_setkey.c optional netsmb crypto/openssl/ossl.c optional ossl +crypto/openssl/ossl_aes.c optional ossl crypto/openssl/ossl_chacha20.c optional ossl crypto/openssl/ossl_poly1305.c optional ossl crypto/openssl/ossl_sha1.c optional ossl diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64 index 37ff6404cdba..6b51c1a5a55d 100644 --- a/sys/conf/files.amd64 +++ b/sys/conf/files.amd64 @@ -88,6 +88,7 @@ cddl/dev/dtrace/amd64/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/amd64/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" crypto/aesni/aeskeys_amd64.S optional aesni crypto/des/des_enc.c optional netsmb +crypto/openssl/amd64/aesni-x86_64.S optional ossl crypto/openssl/amd64/chacha-x86_64.S optional ossl crypto/openssl/amd64/poly1305-x86_64.S optional ossl crypto/openssl/amd64/sha1-x86_64.S optional ossl diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index bb356c1babc1..a2cf15de1ac0 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -115,8 +115,8 @@ armv8_crypto_wrap.o optional armv8crypto \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8/ ${WERROR} ${NO_WCAST_QUAL} -march=armv8-a+crypto ${.IMPSRC}" \ no-implicit-rule \ clean "armv8_crypto_wrap.o" -aesv8-armx.o optional armv8crypto \ - dependency "$S/crypto/openssl/aarch64/aesv8-armx.S" \ +aesv8-armx.o optional armv8crypto | ossl \ + dependency "$S/crypto/openssl/aarch64/aesv8-armx.S" \ compile-with "${CC} -c ${CFLAGS:C/^-O2$/-O3/:N-nostdinc:N-mgeneral-regs-only} -I$S/crypto/armv8/ ${WERROR} ${NO_WCAST_QUAL} -march=armv8-a+crypto ${.IMPSRC}" \ no-implicit-rule \ clean "aesv8-armx.o" @@ -138,6 +138,8 @@ crypto/openssl/aarch64/sha256-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}" crypto/openssl/aarch64/sha512-armv8.S optional ossl \ compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}" +crypto/openssl/aarch64/vpaes-armv8.S optional ossl \ + compile-with "${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}" dev/acpica/acpi_bus_if.m optional acpi dev/acpica/acpi_if.m optional acpi diff --git a/sys/conf/files.i386 b/sys/conf/files.i386 index e83bcd5a3492..0c681d6a84a0 100644 --- a/sys/conf/files.i386 +++ b/sys/conf/files.i386 @@ -15,6 +15,7 @@ cddl/dev/dtrace/i386/dtrace_asm.S optional dtrace compile-with "${DTRACE_S}" cddl/dev/dtrace/i386/dtrace_subr.c optional dtrace compile-with "${DTRACE_C}" crypto/aesni/aeskeys_i386.S optional aesni crypto/des/arch/i386/des_enc.S optional netsmb +crypto/openssl/i386/aesni-x86.S optional ossl crypto/openssl/i386/chacha-x86.S optional ossl crypto/openssl/i386/poly1305-x86.S optional ossl crypto/openssl/i386/sha1-586.S optional ossl diff --git a/sys/crypto/openssl/ossl.c b/sys/crypto/openssl/ossl.c index ad9b93dd960d..f46b5a966bb1 100644 --- a/sys/crypto/openssl/ossl.c +++ b/sys/crypto/openssl/ossl.c @@ -49,24 +49,10 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include "cryptodev_if.h" -struct ossl_softc { - int32_t sc_cid; -}; - -struct ossl_session_hash { - struct ossl_hash_context ictx; - struct ossl_hash_context octx; - struct auth_hash *axf; - u_int mlen; -}; - -struct ossl_session { - struct ossl_session_hash hash; -}; - static MALLOC_DEFINE(M_OSSL, "ossl", "OpenSSL crypto"); static void @@ -92,7 +78,7 @@ ossl_attach(device_t dev) sc = device_get_softc(dev); - ossl_cpuid(); + ossl_cpuid(sc); sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ossl_session), CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC | CRYPTOCAP_F_ACCEL_SOFTWARE); @@ -143,9 +129,34 @@ ossl_lookup_hash(const struct crypto_session_params *csp) } } +static struct ossl_cipher* +ossl_lookup_cipher(const struct crypto_session_params *csp) +{ + + switch (csp->csp_cipher_alg) { + case CRYPTO_AES_CBC: + switch (csp->csp_cipher_klen * 8) { + case 128: + case 192: + case 256: + break; + default: + return (NULL); + } + return (&ossl_cipher_aes_cbc); + case CRYPTO_CHACHA20: + if (csp->csp_cipher_klen != CHACHA_KEY_SIZE) + return (NULL); + return (&ossl_cipher_chacha20); + default: + return (NULL); + } +} + static int ossl_probesession(device_t dev, const struct crypto_session_params *csp) { + struct ossl_softc *sc = device_get_softc(dev); if ((csp->csp_flags & ~(CSP_F_SEPARATE_OUTPUT | CSP_F_SEPARATE_AAD)) != 0) @@ -156,14 +167,10 @@ ossl_probesession(device_t dev, const struct crypto_session_params *csp) return (EINVAL); break; case CSP_MODE_CIPHER: - switch (csp->csp_cipher_alg) { - case CRYPTO_CHACHA20: - if (csp->csp_cipher_klen != CHACHA_KEY_SIZE) - return (EINVAL); - break; - default: + if (csp->csp_cipher_alg != CRYPTO_CHACHA20 && !sc->has_aes) + return (EINVAL); + if (ossl_lookup_cipher(csp) == NULL) return (EINVAL); - } break; case CSP_MODE_AEAD: switch (csp->csp_cipher_alg) { @@ -213,20 +220,57 @@ ossl_newsession_hash(struct ossl_session *s, } } +static int +ossl_newsession_cipher(struct ossl_session *s, + const struct crypto_session_params *csp) +{ + struct ossl_cipher *cipher; + int error = 0; + + cipher = ossl_lookup_cipher(csp); + if (cipher == NULL) + return (EINVAL); + + s->cipher.cipher = cipher; + + if (csp->csp_cipher_key == NULL) + return (0); + + fpu_kern_enter(curthread, NULL, FPU_KERN_NOCTX); + if (cipher->set_encrypt_key != NULL) { + error = cipher->set_encrypt_key(csp->csp_cipher_key, + 8 * csp->csp_cipher_klen, &s->cipher.enc_ctx); + if (error != 0) { + fpu_kern_leave(curthread, NULL); + return (error); + } + } + if (cipher->set_decrypt_key != NULL) + error = cipher->set_decrypt_key(csp->csp_cipher_key, + 8 * csp->csp_cipher_klen, &s->cipher.dec_ctx); + fpu_kern_leave(curthread, NULL); + + return (error); +} + static int ossl_newsession(device_t dev, crypto_session_t cses, const struct crypto_session_params *csp) { struct ossl_session *s; + int error = 0; s = crypto_get_driver_session(cses); switch (csp->csp_mode) { case CSP_MODE_DIGEST: ossl_newsession_hash(s, csp); break; + case CSP_MODE_CIPHER: + error = ossl_newsession_cipher(s, csp); + break; } - return (0); + return (error); } static int @@ -320,7 +364,7 @@ ossl_process(device_t dev, struct cryptop *crp, int hint) error = ossl_process_hash(s, crp, csp); break; case CSP_MODE_CIPHER: - error = ossl_chacha20(crp, csp); + error = s->cipher.cipher->process(&s->cipher, crp, csp); break; case CSP_MODE_AEAD: if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) diff --git a/sys/crypto/openssl/ossl.h b/sys/crypto/openssl/ossl.h index 11793dca037a..4f5353818add 100644 --- a/sys/crypto/openssl/ossl.h +++ b/sys/crypto/openssl/ossl.h @@ -36,20 +36,47 @@ struct cryptop; struct crypto_session_params; +struct ossl_softc; +struct ossl_session; -int ossl_chacha20(struct cryptop *crp, - const struct crypto_session_params *csp); int ossl_chacha20_poly1305_decrypt(struct cryptop *crp, const struct crypto_session_params *csp); int ossl_chacha20_poly1305_encrypt(struct cryptop *crp, const struct crypto_session_params *csp); -void ossl_cpuid(void); +void ossl_cpuid(struct ossl_softc *sc); + +struct ossl_softc { + int32_t sc_cid; + bool has_aes; +}; /* Needs to be big enough to hold any hash context. */ struct ossl_hash_context { uint32_t dummy[61]; } __aligned(32); +struct ossl_cipher_context { + uint32_t dummy[61]; +} __aligned(32); + +struct ossl_session_hash { + struct ossl_hash_context ictx; + struct ossl_hash_context octx; + struct auth_hash *axf; + u_int mlen; +}; + +struct ossl_session_cipher { + struct ossl_cipher_context dec_ctx; + struct ossl_cipher_context enc_ctx; + struct ossl_cipher *cipher; +}; + +struct ossl_session { + struct ossl_session_cipher cipher; + struct ossl_session_hash hash; +}; + extern struct auth_hash ossl_hash_poly1305; extern struct auth_hash ossl_hash_sha1; extern struct auth_hash ossl_hash_sha224; @@ -57,4 +84,7 @@ extern struct auth_hash ossl_hash_sha256; extern struct auth_hash ossl_hash_sha384; extern struct auth_hash ossl_hash_sha512; +extern struct ossl_cipher ossl_cipher_aes_cbc; +extern struct ossl_cipher ossl_cipher_chacha20; + #endif /* !__OSSL_H__ */ diff --git a/sys/crypto/openssl/ossl_aarch64.c b/sys/crypto/openssl/ossl_aarch64.c index 2a45a848808a..e4b87a75a403 100644 --- a/sys/crypto/openssl/ossl_aarch64.c +++ b/sys/crypto/openssl/ossl_aarch64.c @@ -36,6 +36,7 @@ #include #include +#include #include /* @@ -43,8 +44,14 @@ */ unsigned int OPENSSL_armcap_P; +ossl_cipher_setkey_t aes_v8_set_encrypt_key; +ossl_cipher_setkey_t aes_v8_set_decrypt_key; + +ossl_cipher_setkey_t vpaes_set_encrypt_key; +ossl_cipher_setkey_t vpaes_set_decrypt_key; + void -ossl_cpuid(void) +ossl_cpuid(struct ossl_softc *sc) { /* SHA features */ if ((elf_hwcap & HWCAP_SHA1) != 0) @@ -59,4 +66,18 @@ ossl_cpuid(void) OPENSSL_armcap_P |= ARMV8_AES; if ((elf_hwcap & HWCAP_PMULL) != 0) OPENSSL_armcap_P |= ARMV8_PMULL; + + if ((OPENSSL_armcap_P & ARMV8_AES) == 0 && + (OPENSSL_armcap_P & ARMV7_NEON) == 0) { + sc->has_aes = false; + return; + } + sc->has_aes = true; + if (OPENSSL_armcap_P & ARMV8_AES) { + ossl_cipher_aes_cbc.set_encrypt_key = aes_v8_set_encrypt_key; + ossl_cipher_aes_cbc.set_decrypt_key = aes_v8_set_decrypt_key; + } else { + ossl_cipher_aes_cbc.set_encrypt_key = vpaes_set_encrypt_key; + ossl_cipher_aes_cbc.set_decrypt_key = vpaes_set_decrypt_key; + } } diff --git a/sys/crypto/openssl/ossl_aarch64.h b/sys/crypto/openssl/ossl_aarch64.h new file mode 100644 index 000000000000..f933f862d009 --- /dev/null +++ b/sys/crypto/openssl/ossl_aarch64.h @@ -0,0 +1,31 @@ +/* + * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef __OSSL_AARCH64__ +#define __OSSL_AARCH64__ + +#include +#include +#include + +/* aesv8-armx.S */ +ossl_cipher_encrypt_t aes_v8_cbc_encrypt; +/* vpaes-armv8.S */ +ossl_cipher_encrypt_t vpaes_cbc_encrypt; + +static void +AES_CBC_ENCRYPT(const unsigned char *in, unsigned char *out, + size_t length, const void *key, unsigned char *iv, int encrypt) +{ + if (OPENSSL_armcap_P & ARMV8_AES) + aes_v8_cbc_encrypt(in, out, length, key, iv, encrypt); + else + vpaes_cbc_encrypt(in, out, length, key, iv, encrypt); +} +#endif diff --git a/sys/crypto/openssl/ossl_aes.c b/sys/crypto/openssl/ossl_aes.c new file mode 100644 index 000000000000..93f16e7dce55 --- /dev/null +++ b/sys/crypto/openssl/ossl_aes.c @@ -0,0 +1,153 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 Stormshield. + * Copyright (c) 2021 Semihalf. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include + +#include +#include + +#if defined(__amd64__) || defined(__i386__) +#include +#elif defined (__aarch64__) +#include +#endif + +static ossl_cipher_process_t ossl_aes_cbc; + +struct ossl_cipher ossl_cipher_aes_cbc = { + .type = CRYPTO_AES_CBC, + .blocksize = AES_BLOCK_LEN, + .ivsize = AES_BLOCK_LEN, + + /* Filled during initialization based on CPU caps. */ + .set_encrypt_key = NULL, + .set_decrypt_key = NULL, + .process = ossl_aes_cbc +}; + +static int +ossl_aes_cbc(struct ossl_session_cipher *s, struct cryptop *crp, + const struct crypto_session_params *csp) +{ + struct crypto_buffer_cursor cc_in, cc_out; + unsigned char block[EALG_MAX_BLOCK_LEN]; + unsigned char iv[EALG_MAX_BLOCK_LEN]; + const unsigned char *in, *inseg; + unsigned char *out, *outseg; + size_t plen, seglen, inlen, outlen; + struct ossl_cipher_context key; + struct ossl_cipher *cipher; + int blocklen, error; + bool encrypt; + + cipher = s->cipher; + encrypt = CRYPTO_OP_IS_ENCRYPT(crp->crp_op); + plen = crp->crp_payload_length; + blocklen = cipher->blocksize; + + if (plen % blocklen) + return (EINVAL); + + if (crp->crp_cipher_key != NULL) { + if (encrypt) + error = cipher->set_encrypt_key(crp->crp_cipher_key, + 8 * csp->csp_cipher_klen, &key); + else + error = cipher->set_decrypt_key(crp->crp_cipher_key, + 8 * csp->csp_cipher_klen, &key); + if (error) + return (error); + } else { + if (encrypt) + key = s->enc_ctx; + else + key = s->dec_ctx; + } + + crypto_read_iv(crp, iv); + + /* Derived from ossl_chacha20.c */ + crypto_cursor_init(&cc_in, &crp->crp_buf); + crypto_cursor_advance(&cc_in, crp->crp_payload_start); + inseg = crypto_cursor_segment(&cc_in, &inlen); + if (CRYPTO_HAS_OUTPUT_BUFFER(crp)) { + crypto_cursor_init(&cc_out, &crp->crp_obuf); + crypto_cursor_advance(&cc_out, crp->crp_payload_output_start); + } else { + cc_out = cc_in; + } + outseg = crypto_cursor_segment(&cc_out, &outlen); + + while (plen >= blocklen) { + if (inlen < blocklen) { + crypto_cursor_copydata(&cc_in, blocklen, block); + in = block; + inlen = blocklen; + } else { + in = inseg; + } + if (outlen < blocklen) { + out = block; + outlen = blocklen; + } else { + out = outseg; + } + + /* Figure out how many blocks we can encrypt/decrypt at once. */ + seglen = rounddown(MIN(plen, MIN(inlen, outlen)), blocklen); + + AES_CBC_ENCRYPT(in, out, seglen, &key, iv, encrypt); + + if (out == block) { + crypto_cursor_copyback(&cc_out, blocklen, block); + outseg = crypto_cursor_segment(&cc_out, &outlen); + } else { + crypto_cursor_advance(&cc_out, seglen); + outseg += seglen; + outlen -= seglen; + } + if (in == block) { + inseg = crypto_cursor_segment(&cc_in, &inlen); + } else { + crypto_cursor_advance(&cc_in, seglen); + inseg += seglen; + inlen -= seglen; + } + plen -= seglen; + } + + explicit_bzero(block, sizeof(block)); + explicit_bzero(iv, sizeof(iv)); + explicit_bzero(&key, sizeof(key)); + return (0); +} diff --git a/sys/crypto/openssl/ossl_chacha20.c b/sys/crypto/openssl/ossl_chacha20.c index aa125121e8b4..c21a28470a26 100644 --- a/sys/crypto/openssl/ossl_chacha20.c +++ b/sys/crypto/openssl/ossl_chacha20.c @@ -37,10 +37,24 @@ #include #include +#include #include -int -ossl_chacha20(struct cryptop *crp, const struct crypto_session_params *csp) +static ossl_cipher_process_t ossl_chacha20; + +struct ossl_cipher ossl_cipher_chacha20 = { + .type = CRYPTO_CHACHA20, + .blocksize = CHACHA_BLK_SIZE, + .ivsize = CHACHA_CTR_SIZE, + + .set_encrypt_key = NULL, + .set_decrypt_key = NULL, + .process = ossl_chacha20 +}; + +static int +ossl_chacha20(struct ossl_session_cipher *s, struct cryptop *crp, + const struct crypto_session_params *csp) { _Alignas(8) unsigned int key[CHACHA_KEY_SIZE / 4]; unsigned int counter[CHACHA_CTR_SIZE / 4]; diff --git a/sys/crypto/openssl/ossl_cipher.h b/sys/crypto/openssl/ossl_cipher.h new file mode 100644 index 000000000000..d9e6ec29aafc --- /dev/null +++ b/sys/crypto/openssl/ossl_cipher.h @@ -0,0 +1,53 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2021 Stormshield. + * Copyright (c) 2021 Semihalf. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __OSSL_CIPHER_H__ +#define __OSSL_CIPHER_H__ + +struct ossl_session_cipher; +struct cryptop; +struct crypto_session_params; + +typedef int (ossl_cipher_setkey_t)(const unsigned char*, int, void*); +typedef int (ossl_cipher_process_t)(struct ossl_session_cipher*, struct cryptop*, + const struct crypto_session_params*); +typedef void (ossl_cipher_encrypt_t)(const unsigned char*, unsigned char*, size_t, + const void*, unsigned char*, int); + +ossl_cipher_encrypt_t ossl_aes_cbc_encrypt; + +struct ossl_cipher { + int type; + uint16_t blocksize; + uint16_t ivsize; + + ossl_cipher_setkey_t *set_encrypt_key; + ossl_cipher_setkey_t *set_decrypt_key; + ossl_cipher_process_t *process; +}; + +#endif diff --git a/sys/crypto/openssl/ossl_x86.c b/sys/crypto/openssl/ossl_x86.c index 60ff6fa0c759..a1e9a995948b 100644 --- a/sys/crypto/openssl/ossl_x86.c +++ b/sys/crypto/openssl/ossl_x86.c @@ -39,6 +39,7 @@ #include #include +#include /* * See OPENSSL_ia32cap(3). @@ -49,9 +50,13 @@ * [3] = 0 */ unsigned int OPENSSL_ia32cap_P[4]; +#define AESNI_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32))) + +ossl_cipher_setkey_t aesni_set_encrypt_key; +ossl_cipher_setkey_t aesni_set_decrypt_key; void -ossl_cpuid(void) +ossl_cpuid(struct ossl_softc *sc) { uint64_t xcr0; u_int regs[4]; @@ -112,4 +117,12 @@ ossl_cpuid(void) OPENSSL_ia32cap_P[1] &= ~(CPUID2_AVX | AMDID2_XOP | CPUID2_FMA); OPENSSL_ia32cap_P[2] &= ~CPUID_STDEXT_AVX2; } + + if (!AESNI_CAPABLE) { + sc->has_aes = false; + return; + } + sc->has_aes = true; + ossl_cipher_aes_cbc.set_encrypt_key = aesni_set_encrypt_key; + ossl_cipher_aes_cbc.set_decrypt_key = aesni_set_decrypt_key; } diff --git a/sys/crypto/openssl/ossl_x86.h b/sys/crypto/openssl/ossl_x86.h new file mode 100644 index 000000000000..12bd5a4eaddb --- /dev/null +++ b/sys/crypto/openssl/ossl_x86.h @@ -0,0 +1,20 @@ +/* + * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved. + * + * Licensed under the OpenSSL license (the "License"). You may not use + * this file except in compliance with the License. You can obtain a copy + * in the file LICENSE in the source distribution or at + * https://www.openssl.org/source/license.html + */ + +#ifndef __OSSL_X86__ +#define __OSSL_X86__ + +#include +#include + +/* aesni-x86_64.S, aesni-x86.S */ +ossl_cipher_encrypt_t aesni_cbc_encrypt; + +#define AES_CBC_ENCRYPT aesni_cbc_encrypt +#endif diff --git a/sys/modules/ossl/Makefile b/sys/modules/ossl/Makefile index dfd82dcf6e1f..765e70a03edd 100644 --- a/sys/modules/ossl/Makefile +++ b/sys/modules/ossl/Makefile @@ -4,10 +4,12 @@ .PATH: ${SRCTOP}/sys/crypto/openssl/${MACHINE_CPUARCH} KMOD= ossl +OBJS+= ${OBJS.${MACHINE_CPUARCH}} SRCS= bus_if.h \ cryptodev_if.h \ device_if.h \ ossl.c \ + ossl_aes.c \ ossl_chacha20.c \ ossl_poly1305.c \ ossl_sha1.c \ @@ -21,9 +23,11 @@ SRCS.aarch64= \ sha1-armv8.S \ sha256-armv8.S \ sha512-armv8.S \ + vpaes-armv8.S \ ossl_aarch64.c SRCS.amd64= \ + aesni-x86_64.S \ chacha-x86_64.S \ poly1305-x86_64.S \ sha1-x86_64.S \ @@ -32,6 +36,7 @@ SRCS.amd64= \ ossl_x86.c SRCS.i386= \ + aesni-x86.S \ chacha-x86.S \ poly1305-x86.S \ sha1-586.S \ @@ -45,4 +50,13 @@ ${SRCS.aarch64:M*.S:S/S/o/}: ${.TARGET:R}.S ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} ${.IMPSRC} ${CTFCONVERT_CMD} +# Based on modules/armv8crypto/Makefile. +# Clang doesn't recognize "aes*" instructions without -march set. +aesv8-armx.o: aesv8-armx.S + ${CC} -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${PROF} \ + -march=armv8-a+crypto ${.IMPSRC} + ${CTFCONVERT_CMD} + +OBJS.aarch64= aesv8-armx.o + .include diff --git a/tests/sys/opencrypto/cryptotest.py b/tests/sys/opencrypto/cryptotest.py index 74ce62cee33d..447a7854b246 100644 --- a/tests/sys/opencrypto/cryptotest.py +++ b/tests/sys/opencrypto/cryptotest.py @@ -50,7 +50,7 @@ def katg(base, glob): raise unittest.SkipTest("Missing %s test vectors" % (base)) return iglob(os.path.join(katdir, base, glob)) -aesmodules = [ 'cryptosoft0', 'aesni0', 'armv8crypto0', 'ccr0', 'ccp0', 'safexcel0', 'qat0' ] +aesmodules = [ 'cryptosoft0', 'aesni0', 'armv8crypto0', 'ccr0', 'ccp0', 'ossl0', 'safexcel0', 'qat0' ] shamodules = [ 'cryptosoft0', 'aesni0', 'armv8crypto0', 'ccr0', 'ccp0', 'ossl0', 'safexcel0', 'qat0' ] def GenTestCase(cname):