armv8crypto: add AES-XTS support
A straightforward(ish) port from aesni(4). This implementation does not perform loop unrolling on the input blocks, so this is left as a future performance improvement. Submitted by: Greg V <greg AT unrelenting.technology> Looks good: jhb, jmg Tested by: mhorne Differential Revision: https://reviews.freebsd.org/D21017
This commit is contained in:
parent
1790f5e654
commit
4979620ece
@ -114,7 +114,7 @@ armv8_crypto_probe(device_t dev)
|
||||
break;
|
||||
}
|
||||
|
||||
device_set_desc_copy(dev, "AES-CBC");
|
||||
device_set_desc_copy(dev, "AES-CBC,AES-XTS");
|
||||
|
||||
/* TODO: Check more fields as we support more features */
|
||||
|
||||
@ -204,6 +204,17 @@ armv8_crypto_probesession(device_t dev,
|
||||
return (EINVAL);
|
||||
}
|
||||
break;
|
||||
case CRYPTO_AES_XTS:
|
||||
if (csp->csp_ivlen != AES_XTS_IV_LEN)
|
||||
return (EINVAL);
|
||||
switch (csp->csp_cipher_klen * 8) {
|
||||
case 256:
|
||||
case 512:
|
||||
break;
|
||||
default:
|
||||
return (EINVAL);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return (EINVAL);
|
||||
}
|
||||
@ -211,16 +222,19 @@ armv8_crypto_probesession(device_t dev,
|
||||
default:
|
||||
return (EINVAL);
|
||||
}
|
||||
return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
|
||||
return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
|
||||
}
|
||||
|
||||
static void
|
||||
armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
|
||||
const struct crypto_session_params *csp)
|
||||
const struct crypto_session_params *csp, const uint8_t *key, int keylen)
|
||||
{
|
||||
int i;
|
||||
|
||||
switch (csp->csp_cipher_klen * 8) {
|
||||
if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
|
||||
keylen /= 2;
|
||||
|
||||
switch (keylen * 8) {
|
||||
case 128:
|
||||
ses->rounds = AES128_ROUNDS;
|
||||
break;
|
||||
@ -231,16 +245,19 @@ armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
|
||||
ses->rounds = AES256_ROUNDS;
|
||||
break;
|
||||
default:
|
||||
panic("invalid CBC key length");
|
||||
panic("invalid AES key length");
|
||||
}
|
||||
|
||||
rijndaelKeySetupEnc(ses->enc_schedule, csp->csp_cipher_key,
|
||||
csp->csp_cipher_klen * 8);
|
||||
rijndaelKeySetupDec(ses->dec_schedule, csp->csp_cipher_key,
|
||||
csp->csp_cipher_klen * 8);
|
||||
rijndaelKeySetupEnc(ses->enc_schedule, key, keylen * 8);
|
||||
rijndaelKeySetupDec(ses->dec_schedule, key, keylen * 8);
|
||||
if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
|
||||
rijndaelKeySetupEnc(ses->xts_schedule, key + keylen, keylen * 8);
|
||||
|
||||
for (i = 0; i < nitems(ses->enc_schedule); i++) {
|
||||
ses->enc_schedule[i] = bswap32(ses->enc_schedule[i]);
|
||||
ses->dec_schedule[i] = bswap32(ses->dec_schedule[i]);
|
||||
if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
|
||||
ses->xts_schedule[i] = bswap32(ses->xts_schedule[i]);
|
||||
}
|
||||
}
|
||||
|
||||
@ -259,7 +276,8 @@ armv8_crypto_newsession(device_t dev, crypto_session_t cses,
|
||||
}
|
||||
|
||||
ses = crypto_get_driver_session(cses);
|
||||
armv8_crypto_cipher_setup(ses, csp);
|
||||
armv8_crypto_cipher_setup(ses, csp, csp->csp_cipher_key,
|
||||
csp->csp_cipher_klen);
|
||||
rw_wunlock(&sc->lock);
|
||||
return (0);
|
||||
}
|
||||
@ -333,7 +351,8 @@ armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
|
||||
}
|
||||
|
||||
if (crp->crp_cipher_key != NULL) {
|
||||
panic("armv8: new cipher key");
|
||||
armv8_crypto_cipher_setup(ses, csp, crp->crp_cipher_key,
|
||||
csp->csp_cipher_klen);
|
||||
}
|
||||
|
||||
crypto_read_iv(crp, iv);
|
||||
@ -348,6 +367,16 @@ armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
|
||||
armv8_aes_decrypt_cbc(ses->rounds, ses->dec_schedule,
|
||||
crp->crp_payload_length, buf, iv);
|
||||
break;
|
||||
case CRYPTO_AES_XTS:
|
||||
if (encflag)
|
||||
armv8_aes_encrypt_xts(ses->rounds, ses->enc_schedule,
|
||||
ses->xts_schedule, crp->crp_payload_length, buf,
|
||||
buf, iv);
|
||||
else
|
||||
armv8_aes_decrypt_xts(ses->rounds, ses->dec_schedule,
|
||||
ses->xts_schedule, crp->crp_payload_length, buf,
|
||||
buf, iv);
|
||||
break;
|
||||
}
|
||||
|
||||
if (allocated)
|
||||
|
@ -40,6 +40,7 @@
|
||||
struct armv8_crypto_session {
|
||||
uint32_t enc_schedule[AES_SCHED_LEN/4];
|
||||
uint32_t dec_schedule[AES_SCHED_LEN/4];
|
||||
uint32_t xts_schedule[AES_SCHED_LEN/4];
|
||||
int algo;
|
||||
int rounds;
|
||||
};
|
||||
@ -49,4 +50,9 @@ void armv8_aes_encrypt_cbc(int, const void *, size_t, const uint8_t *,
|
||||
void armv8_aes_decrypt_cbc(int, const void *, size_t, uint8_t *,
|
||||
const uint8_t[static AES_BLOCK_LEN]);
|
||||
|
||||
void armv8_aes_encrypt_xts(int, const void *, const void *, size_t,
|
||||
const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]);
|
||||
void armv8_aes_decrypt_xts(int, const void *, const void *, size_t,
|
||||
const uint8_t *, uint8_t *, const uint8_t[AES_BLOCK_LEN]);
|
||||
|
||||
#endif /* _ARMV8_CRYPTO_H_ */
|
||||
|
@ -126,3 +126,95 @@ armv8_aes_decrypt_cbc(int rounds, const void *key_schedule, size_t len,
|
||||
buf += AES_BLOCK_LEN;
|
||||
}
|
||||
}
|
||||
|
||||
#define AES_XTS_BLOCKSIZE 16
|
||||
#define AES_XTS_IVSIZE 8
|
||||
#define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */
|
||||
|
||||
static inline int32x4_t
|
||||
xts_crank_lfsr(int32x4_t inp)
|
||||
{
|
||||
const int32x4_t alphamask = {AES_XTS_ALPHA, 1, 1, 1};
|
||||
int32x4_t xtweak, ret;
|
||||
|
||||
/* set up xor mask */
|
||||
xtweak = vextq_s32(inp, inp, 3);
|
||||
xtweak = vshrq_n_s32(xtweak, 31);
|
||||
xtweak &= alphamask;
|
||||
|
||||
/* next term */
|
||||
ret = vshlq_n_s32(inp, 1);
|
||||
ret ^= xtweak;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void
|
||||
armv8_aes_crypt_xts_block(int rounds, const uint8x16_t *key_schedule,
|
||||
uint8x16_t *tweak, const uint8_t *from, uint8_t *to, int do_encrypt)
|
||||
{
|
||||
uint8x16_t block;
|
||||
|
||||
block = vld1q_u8(from) ^ *tweak;
|
||||
|
||||
if (do_encrypt)
|
||||
block = armv8_aes_enc(rounds - 1, key_schedule, block);
|
||||
else
|
||||
block = armv8_aes_dec(rounds - 1, key_schedule, block);
|
||||
|
||||
vst1q_u8(to, block ^ *tweak);
|
||||
|
||||
*tweak = vreinterpretq_u8_s32(xts_crank_lfsr(vreinterpretq_s32_u8(*tweak)));
|
||||
}
|
||||
|
||||
static void
|
||||
armv8_aes_crypt_xts(int rounds, const uint8x16_t *data_schedule,
|
||||
const uint8x16_t *tweak_schedule, size_t len, const uint8_t *from,
|
||||
uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN], int do_encrypt)
|
||||
{
|
||||
uint8x16_t tweakreg;
|
||||
uint8_t tweak[AES_XTS_BLOCKSIZE] __aligned(16);
|
||||
size_t i, cnt;
|
||||
|
||||
/*
|
||||
* Prepare tweak as E_k2(IV). IV is specified as LE representation
|
||||
* of a 64-bit block number which we allow to be passed in directly.
|
||||
*/
|
||||
#if BYTE_ORDER == LITTLE_ENDIAN
|
||||
bcopy(iv, tweak, AES_XTS_IVSIZE);
|
||||
/* Last 64 bits of IV are always zero. */
|
||||
bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE);
|
||||
#else
|
||||
#error Only LITTLE_ENDIAN architectures are supported.
|
||||
#endif
|
||||
tweakreg = vld1q_u8(tweak);
|
||||
tweakreg = armv8_aes_enc(rounds - 1, tweak_schedule, tweakreg);
|
||||
|
||||
cnt = len / AES_XTS_BLOCKSIZE;
|
||||
for (i = 0; i < cnt; i++) {
|
||||
armv8_aes_crypt_xts_block(rounds, data_schedule, &tweakreg,
|
||||
from, to, do_encrypt);
|
||||
from += AES_XTS_BLOCKSIZE;
|
||||
to += AES_XTS_BLOCKSIZE;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
armv8_aes_encrypt_xts(int rounds, const void *data_schedule,
|
||||
const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
|
||||
const uint8_t iv[static AES_BLOCK_LEN])
|
||||
{
|
||||
|
||||
armv8_aes_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
|
||||
iv, 1);
|
||||
}
|
||||
|
||||
void
|
||||
armv8_aes_decrypt_xts(int rounds, const void *data_schedule,
|
||||
const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
|
||||
const uint8_t iv[static AES_BLOCK_LEN])
|
||||
{
|
||||
|
||||
armv8_aes_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
|
||||
iv, 0);
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user