Don't dynamically allocate data structures for KTLS crypto requests.
Allocate iovec arrays and struct cryptop and struct ocf_operation objects on the stack to reduce avoid the overhead of malloc(). These structures are all small enough to fit on the stack of the KTLS worker threads. Reviewed by: gallatin Sponsored by: Netflix Differential Revision: https://reviews.freebsd.org/D25692
This commit is contained in:
parent
ac11d85740
commit
33a1a488d5
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=363379
@ -51,7 +51,6 @@ struct ocf_session {
|
|||||||
struct ocf_operation {
|
struct ocf_operation {
|
||||||
struct ocf_session *os;
|
struct ocf_session *os;
|
||||||
bool done;
|
bool done;
|
||||||
struct iovec iov[0];
|
|
||||||
};
|
};
|
||||||
|
|
||||||
static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS");
|
static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS");
|
||||||
@ -109,18 +108,18 @@ ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls,
|
|||||||
{
|
{
|
||||||
struct uio uio, out_uio, *tag_uio;
|
struct uio uio, out_uio, *tag_uio;
|
||||||
struct tls_aead_data ad;
|
struct tls_aead_data ad;
|
||||||
struct cryptop *crp;
|
struct cryptop crp;
|
||||||
struct ocf_session *os;
|
struct ocf_session *os;
|
||||||
struct ocf_operation *oo;
|
struct ocf_operation oo;
|
||||||
|
struct iovec iov[iovcnt + 1];
|
||||||
int i, error;
|
int i, error;
|
||||||
uint16_t tls_comp_len;
|
uint16_t tls_comp_len;
|
||||||
bool inplace;
|
bool inplace;
|
||||||
|
|
||||||
os = tls->cipher;
|
os = tls->cipher;
|
||||||
|
|
||||||
oo = malloc(sizeof(*oo) + (iovcnt + 1) * sizeof(struct iovec),
|
oo.os = os;
|
||||||
M_KTLS_OCF, M_WAITOK | M_ZERO);
|
oo.done = false;
|
||||||
oo->os = os;
|
|
||||||
|
|
||||||
uio.uio_iov = iniov;
|
uio.uio_iov = iniov;
|
||||||
uio.uio_iovcnt = iovcnt;
|
uio.uio_iovcnt = iovcnt;
|
||||||
@ -134,11 +133,11 @@ ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls,
|
|||||||
out_uio.uio_segflg = UIO_SYSSPACE;
|
out_uio.uio_segflg = UIO_SYSSPACE;
|
||||||
out_uio.uio_td = curthread;
|
out_uio.uio_td = curthread;
|
||||||
|
|
||||||
crp = crypto_getreq(os->sid, M_WAITOK);
|
crypto_initreq(&crp, os->sid);
|
||||||
|
|
||||||
/* Setup the IV. */
|
/* Setup the IV. */
|
||||||
memcpy(crp->crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
|
memcpy(crp.crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
|
||||||
memcpy(crp->crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t));
|
memcpy(crp.crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t));
|
||||||
|
|
||||||
/* Setup the AAD. */
|
/* Setup the AAD. */
|
||||||
tls_comp_len = ntohs(hdr->tls_length) -
|
tls_comp_len = ntohs(hdr->tls_length) -
|
||||||
@ -148,19 +147,19 @@ ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls,
|
|||||||
ad.tls_vmajor = hdr->tls_vmajor;
|
ad.tls_vmajor = hdr->tls_vmajor;
|
||||||
ad.tls_vminor = hdr->tls_vminor;
|
ad.tls_vminor = hdr->tls_vminor;
|
||||||
ad.tls_length = htons(tls_comp_len);
|
ad.tls_length = htons(tls_comp_len);
|
||||||
crp->crp_aad = &ad;
|
crp.crp_aad = &ad;
|
||||||
crp->crp_aad_length = sizeof(ad);
|
crp.crp_aad_length = sizeof(ad);
|
||||||
|
|
||||||
/* Compute payload length and determine if encryption is in place. */
|
/* Compute payload length and determine if encryption is in place. */
|
||||||
inplace = true;
|
inplace = true;
|
||||||
crp->crp_payload_start = 0;
|
crp.crp_payload_start = 0;
|
||||||
for (i = 0; i < iovcnt; i++) {
|
for (i = 0; i < iovcnt; i++) {
|
||||||
if (iniov[i].iov_base != outiov[i].iov_base)
|
if (iniov[i].iov_base != outiov[i].iov_base)
|
||||||
inplace = false;
|
inplace = false;
|
||||||
crp->crp_payload_length += iniov[i].iov_len;
|
crp.crp_payload_length += iniov[i].iov_len;
|
||||||
}
|
}
|
||||||
uio.uio_resid = crp->crp_payload_length;
|
uio.uio_resid = crp.crp_payload_length;
|
||||||
out_uio.uio_resid = crp->crp_payload_length;
|
out_uio.uio_resid = crp.crp_payload_length;
|
||||||
|
|
||||||
if (inplace)
|
if (inplace)
|
||||||
tag_uio = &uio;
|
tag_uio = &uio;
|
||||||
@ -168,21 +167,21 @@ ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls,
|
|||||||
tag_uio = &out_uio;
|
tag_uio = &out_uio;
|
||||||
|
|
||||||
/* Duplicate iovec and append vector for tag. */
|
/* Duplicate iovec and append vector for tag. */
|
||||||
memcpy(oo->iov, tag_uio->uio_iov, iovcnt * sizeof(struct iovec));
|
memcpy(iov, tag_uio->uio_iov, iovcnt * sizeof(struct iovec));
|
||||||
tag_uio->uio_iov = oo->iov;
|
iov[iovcnt].iov_base = trailer;
|
||||||
tag_uio->uio_iov[iovcnt].iov_base = trailer;
|
iov[iovcnt].iov_len = AES_GMAC_HASH_LEN;
|
||||||
tag_uio->uio_iov[iovcnt].iov_len = AES_GMAC_HASH_LEN;
|
tag_uio->uio_iov = iov;
|
||||||
tag_uio->uio_iovcnt++;
|
tag_uio->uio_iovcnt++;
|
||||||
crp->crp_digest_start = tag_uio->uio_resid;
|
crp.crp_digest_start = tag_uio->uio_resid;
|
||||||
tag_uio->uio_resid += AES_GMAC_HASH_LEN;
|
tag_uio->uio_resid += AES_GMAC_HASH_LEN;
|
||||||
|
|
||||||
crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
|
crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
|
||||||
crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
|
crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
|
||||||
crypto_use_uio(crp, &uio);
|
crypto_use_uio(&crp, &uio);
|
||||||
if (!inplace)
|
if (!inplace)
|
||||||
crypto_use_output_uio(crp, &out_uio);
|
crypto_use_output_uio(&crp, &out_uio);
|
||||||
crp->crp_opaque = oo;
|
crp.crp_opaque = &oo;
|
||||||
crp->crp_callback = ktls_ocf_callback;
|
crp.crp_callback = ktls_ocf_callback;
|
||||||
|
|
||||||
counter_u64_add(ocf_tls12_gcm_crypts, 1);
|
counter_u64_add(ocf_tls12_gcm_crypts, 1);
|
||||||
if (inplace)
|
if (inplace)
|
||||||
@ -190,28 +189,27 @@ ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls,
|
|||||||
else
|
else
|
||||||
counter_u64_add(ocf_separate_output, 1);
|
counter_u64_add(ocf_separate_output, 1);
|
||||||
for (;;) {
|
for (;;) {
|
||||||
error = crypto_dispatch(crp);
|
error = crypto_dispatch(&crp);
|
||||||
if (error)
|
if (error)
|
||||||
break;
|
break;
|
||||||
|
|
||||||
mtx_lock(&os->lock);
|
mtx_lock(&os->lock);
|
||||||
while (!oo->done)
|
while (!oo.done)
|
||||||
mtx_sleep(oo, &os->lock, 0, "ocfktls", 0);
|
mtx_sleep(&oo, &os->lock, 0, "ocfktls", 0);
|
||||||
mtx_unlock(&os->lock);
|
mtx_unlock(&os->lock);
|
||||||
|
|
||||||
if (crp->crp_etype != EAGAIN) {
|
if (crp.crp_etype != EAGAIN) {
|
||||||
error = crp->crp_etype;
|
error = crp.crp_etype;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
crp->crp_etype = 0;
|
crp.crp_etype = 0;
|
||||||
crp->crp_flags &= ~CRYPTO_F_DONE;
|
crp.crp_flags &= ~CRYPTO_F_DONE;
|
||||||
oo->done = false;
|
oo.done = false;
|
||||||
counter_u64_add(ocf_retries, 1);
|
counter_u64_add(ocf_retries, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
crypto_freereq(crp);
|
crypto_destroyreq(&crp);
|
||||||
free(oo, M_KTLS_OCF);
|
|
||||||
return (error);
|
return (error);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -223,22 +221,19 @@ ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls,
|
|||||||
struct uio uio, out_uio;
|
struct uio uio, out_uio;
|
||||||
struct tls_aead_data_13 ad;
|
struct tls_aead_data_13 ad;
|
||||||
char nonce[12];
|
char nonce[12];
|
||||||
struct cryptop *crp;
|
struct cryptop crp;
|
||||||
struct ocf_session *os;
|
struct ocf_session *os;
|
||||||
struct ocf_operation *oo;
|
struct ocf_operation oo;
|
||||||
struct iovec *iov, *out_iov;
|
struct iovec iov[iovcnt + 1], out_iov[iovcnt + 1];
|
||||||
int i, error;
|
int i, error;
|
||||||
bool inplace;
|
bool inplace;
|
||||||
|
|
||||||
os = tls->cipher;
|
os = tls->cipher;
|
||||||
|
|
||||||
oo = malloc(sizeof(*oo) + (iovcnt + 1) * sizeof(*iov) * 2, M_KTLS_OCF,
|
oo.os = os;
|
||||||
M_WAITOK | M_ZERO);
|
oo.done = false;
|
||||||
oo->os = os;
|
|
||||||
iov = oo->iov;
|
|
||||||
out_iov = iov + iovcnt + 2;
|
|
||||||
|
|
||||||
crp = crypto_getreq(os->sid, M_WAITOK);
|
crypto_initreq(&crp, os->sid);
|
||||||
|
|
||||||
/* Setup the nonce. */
|
/* Setup the nonce. */
|
||||||
memcpy(nonce, tls->params.iv, tls->params.iv_len);
|
memcpy(nonce, tls->params.iv, tls->params.iv_len);
|
||||||
@ -249,22 +244,22 @@ ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls,
|
|||||||
ad.tls_vmajor = hdr->tls_vmajor;
|
ad.tls_vmajor = hdr->tls_vmajor;
|
||||||
ad.tls_vminor = hdr->tls_vminor;
|
ad.tls_vminor = hdr->tls_vminor;
|
||||||
ad.tls_length = hdr->tls_length;
|
ad.tls_length = hdr->tls_length;
|
||||||
crp->crp_aad = &ad;
|
crp.crp_aad = &ad;
|
||||||
crp->crp_aad_length = sizeof(ad);
|
crp.crp_aad_length = sizeof(ad);
|
||||||
|
|
||||||
/* Compute payload length and determine if encryption is in place. */
|
/* Compute payload length and determine if encryption is in place. */
|
||||||
inplace = true;
|
inplace = true;
|
||||||
crp->crp_payload_start = 0;
|
crp.crp_payload_start = 0;
|
||||||
for (i = 0; i < iovcnt; i++) {
|
for (i = 0; i < iovcnt; i++) {
|
||||||
if (iniov[i].iov_base != outiov[i].iov_base)
|
if (iniov[i].iov_base != outiov[i].iov_base)
|
||||||
inplace = false;
|
inplace = false;
|
||||||
crp->crp_payload_length += iniov[i].iov_len;
|
crp.crp_payload_length += iniov[i].iov_len;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Store the record type as the first byte of the trailer. */
|
/* Store the record type as the first byte of the trailer. */
|
||||||
trailer[0] = record_type;
|
trailer[0] = record_type;
|
||||||
crp->crp_payload_length++;
|
crp.crp_payload_length++;
|
||||||
crp->crp_digest_start = crp->crp_payload_length;
|
crp.crp_digest_start = crp.crp_payload_length;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Duplicate the input iov to append the trailer. Always
|
* Duplicate the input iov to append the trailer. Always
|
||||||
@ -277,10 +272,10 @@ ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls,
|
|||||||
uio.uio_iov = iov;
|
uio.uio_iov = iov;
|
||||||
uio.uio_iovcnt = iovcnt + 1;
|
uio.uio_iovcnt = iovcnt + 1;
|
||||||
uio.uio_offset = 0;
|
uio.uio_offset = 0;
|
||||||
uio.uio_resid = crp->crp_payload_length + AES_GMAC_HASH_LEN;
|
uio.uio_resid = crp.crp_payload_length + AES_GMAC_HASH_LEN;
|
||||||
uio.uio_segflg = UIO_SYSSPACE;
|
uio.uio_segflg = UIO_SYSSPACE;
|
||||||
uio.uio_td = curthread;
|
uio.uio_td = curthread;
|
||||||
crypto_use_uio(crp, &uio);
|
crypto_use_uio(&crp, &uio);
|
||||||
|
|
||||||
if (!inplace) {
|
if (!inplace) {
|
||||||
/* Duplicate the output iov to append the trailer. */
|
/* Duplicate the output iov to append the trailer. */
|
||||||
@ -290,19 +285,19 @@ ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls,
|
|||||||
out_uio.uio_iov = out_iov;
|
out_uio.uio_iov = out_iov;
|
||||||
out_uio.uio_iovcnt = iovcnt + 1;
|
out_uio.uio_iovcnt = iovcnt + 1;
|
||||||
out_uio.uio_offset = 0;
|
out_uio.uio_offset = 0;
|
||||||
out_uio.uio_resid = crp->crp_payload_length +
|
out_uio.uio_resid = crp.crp_payload_length +
|
||||||
AES_GMAC_HASH_LEN;
|
AES_GMAC_HASH_LEN;
|
||||||
out_uio.uio_segflg = UIO_SYSSPACE;
|
out_uio.uio_segflg = UIO_SYSSPACE;
|
||||||
out_uio.uio_td = curthread;
|
out_uio.uio_td = curthread;
|
||||||
crypto_use_output_uio(crp, &out_uio);
|
crypto_use_output_uio(&crp, &out_uio);
|
||||||
}
|
}
|
||||||
|
|
||||||
crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
|
crp.crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
|
||||||
crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
|
crp.crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
|
||||||
crp->crp_opaque = oo;
|
crp.crp_opaque = &oo;
|
||||||
crp->crp_callback = ktls_ocf_callback;
|
crp.crp_callback = ktls_ocf_callback;
|
||||||
|
|
||||||
memcpy(crp->crp_iv, nonce, sizeof(nonce));
|
memcpy(crp.crp_iv, nonce, sizeof(nonce));
|
||||||
|
|
||||||
counter_u64_add(ocf_tls13_gcm_crypts, 1);
|
counter_u64_add(ocf_tls13_gcm_crypts, 1);
|
||||||
if (inplace)
|
if (inplace)
|
||||||
@ -310,28 +305,27 @@ ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls,
|
|||||||
else
|
else
|
||||||
counter_u64_add(ocf_separate_output, 1);
|
counter_u64_add(ocf_separate_output, 1);
|
||||||
for (;;) {
|
for (;;) {
|
||||||
error = crypto_dispatch(crp);
|
error = crypto_dispatch(&crp);
|
||||||
if (error)
|
if (error)
|
||||||
break;
|
break;
|
||||||
|
|
||||||
mtx_lock(&os->lock);
|
mtx_lock(&os->lock);
|
||||||
while (!oo->done)
|
while (!oo.done)
|
||||||
mtx_sleep(oo, &os->lock, 0, "ocfktls", 0);
|
mtx_sleep(&oo, &os->lock, 0, "ocfktls", 0);
|
||||||
mtx_unlock(&os->lock);
|
mtx_unlock(&os->lock);
|
||||||
|
|
||||||
if (crp->crp_etype != EAGAIN) {
|
if (crp.crp_etype != EAGAIN) {
|
||||||
error = crp->crp_etype;
|
error = crp.crp_etype;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
crp->crp_etype = 0;
|
crp.crp_etype = 0;
|
||||||
crp->crp_flags &= ~CRYPTO_F_DONE;
|
crp.crp_flags &= ~CRYPTO_F_DONE;
|
||||||
oo->done = false;
|
oo.done = false;
|
||||||
counter_u64_add(ocf_retries, 1);
|
counter_u64_add(ocf_retries, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
crypto_freereq(crp);
|
crypto_destroyreq(&crp);
|
||||||
free(oo, M_KTLS_OCF);
|
|
||||||
return (error);
|
return (error);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user