ktls: Cache output buffers for software encryption
Maintain a cache of physically contiguous runs of pages for use as output buffers when software encryption is configured and in-place encryption is not possible. This makes allocation and free cheaper since in the common case we avoid touching the vm_page structures for the buffer, and fewer calls into UMA are needed. gallatin@ reports a ~10% absolute decrease in CPU usage with sendfile/KTLS on a Xeon after this change. It is possible that we will not be able to allocate these buffers if physical memory is fragmented. To avoid frequently calling into the physical memory allocator in this scenario, rate-limit allocation attempts after a failure. In the failure case we fall back to the old behaviour of allocating a page at a time. N.B.: this scheme could be simplified, either by simply using malloc() and looking up the PAs of the pages backing the buffer, or by falling back to page by page allocation and creating a mapping in the cache zone. This requires some way to save a mapping of an M_EXTPG page array in the mbuf, though. m_data is not really appropriate. The second approach may be possible by saving the mapping in the plinks union of the first vm_page structure of the array, but this would force a vm_page access when freeing an mbuf. Reviewed by: gallatin, jhb Tested by: gallatin Sponsored by: Ampere Computing Submitted by: Klara, Inc. Differential Revision: https://reviews.freebsd.org/D28556
This commit is contained in:
parent
c113740f26
commit
49f6925ca3
@ -82,6 +82,7 @@ struct ktls_wq {
|
|||||||
STAILQ_HEAD(, mbuf) m_head;
|
STAILQ_HEAD(, mbuf) m_head;
|
||||||
STAILQ_HEAD(, socket) so_head;
|
STAILQ_HEAD(, socket) so_head;
|
||||||
bool running;
|
bool running;
|
||||||
|
int lastallocfail;
|
||||||
} __aligned(CACHE_LINE_SIZE);
|
} __aligned(CACHE_LINE_SIZE);
|
||||||
|
|
||||||
struct ktls_domain_info {
|
struct ktls_domain_info {
|
||||||
@ -95,6 +96,7 @@ static struct proc *ktls_proc;
|
|||||||
LIST_HEAD(, ktls_crypto_backend) ktls_backends;
|
LIST_HEAD(, ktls_crypto_backend) ktls_backends;
|
||||||
static struct rmlock ktls_backends_lock;
|
static struct rmlock ktls_backends_lock;
|
||||||
static uma_zone_t ktls_session_zone;
|
static uma_zone_t ktls_session_zone;
|
||||||
|
static uma_zone_t ktls_buffer_zone;
|
||||||
static uint16_t ktls_cpuid_lookup[MAXCPU];
|
static uint16_t ktls_cpuid_lookup[MAXCPU];
|
||||||
|
|
||||||
SYSCTL_NODE(_kern_ipc, OID_AUTO, tls, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
|
SYSCTL_NODE(_kern_ipc, OID_AUTO, tls, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
|
||||||
@ -116,7 +118,7 @@ SYSCTL_INT(_kern_ipc_tls, OID_AUTO, bind_threads, CTLFLAG_RDTUN,
|
|||||||
"Bind crypto threads to cores (1) or cores and domains (2) at boot");
|
"Bind crypto threads to cores (1) or cores and domains (2) at boot");
|
||||||
|
|
||||||
static u_int ktls_maxlen = 16384;
|
static u_int ktls_maxlen = 16384;
|
||||||
SYSCTL_UINT(_kern_ipc_tls, OID_AUTO, maxlen, CTLFLAG_RWTUN,
|
SYSCTL_UINT(_kern_ipc_tls, OID_AUTO, maxlen, CTLFLAG_RDTUN,
|
||||||
&ktls_maxlen, 0, "Maximum TLS record size");
|
&ktls_maxlen, 0, "Maximum TLS record size");
|
||||||
|
|
||||||
static int ktls_number_threads;
|
static int ktls_number_threads;
|
||||||
@ -134,6 +136,11 @@ SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, cbc_enable, CTLFLAG_RWTUN,
|
|||||||
&ktls_cbc_enable, 1,
|
&ktls_cbc_enable, 1,
|
||||||
"Enable Support of AES-CBC crypto for kernel TLS");
|
"Enable Support of AES-CBC crypto for kernel TLS");
|
||||||
|
|
||||||
|
static bool ktls_sw_buffer_cache = true;
|
||||||
|
SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, sw_buffer_cache, CTLFLAG_RDTUN,
|
||||||
|
&ktls_sw_buffer_cache, 1,
|
||||||
|
"Enable caching of output buffers for SW encryption");
|
||||||
|
|
||||||
static COUNTER_U64_DEFINE_EARLY(ktls_tasks_active);
|
static COUNTER_U64_DEFINE_EARLY(ktls_tasks_active);
|
||||||
SYSCTL_COUNTER_U64(_kern_ipc_tls, OID_AUTO, tasks_active, CTLFLAG_RD,
|
SYSCTL_COUNTER_U64(_kern_ipc_tls, OID_AUTO, tasks_active, CTLFLAG_RD,
|
||||||
&ktls_tasks_active, "Number of active tasks");
|
&ktls_tasks_active, "Number of active tasks");
|
||||||
@ -366,6 +373,51 @@ ktls_get_cpu(struct socket *so)
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
static int
|
||||||
|
ktls_buffer_import(void *arg, void **store, int count, int domain, int flags)
|
||||||
|
{
|
||||||
|
vm_page_t m;
|
||||||
|
int i;
|
||||||
|
|
||||||
|
KASSERT((ktls_maxlen & PAGE_MASK) == 0,
|
||||||
|
("%s: ktls max length %d is not page size-aligned",
|
||||||
|
__func__, ktls_maxlen));
|
||||||
|
|
||||||
|
for (i = 0; i < count; i++) {
|
||||||
|
m = vm_page_alloc_contig_domain(NULL, 0, domain,
|
||||||
|
VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED |
|
||||||
|
VM_ALLOC_NODUMP | malloc2vm_flags(flags),
|
||||||
|
atop(ktls_maxlen), 0, ~0ul, PAGE_SIZE, 0,
|
||||||
|
VM_MEMATTR_DEFAULT);
|
||||||
|
if (m == NULL)
|
||||||
|
break;
|
||||||
|
store[i] = (void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
|
||||||
|
}
|
||||||
|
return (i);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
ktls_buffer_release(void *arg __unused, void **store, int count)
|
||||||
|
{
|
||||||
|
vm_page_t m;
|
||||||
|
int i, j;
|
||||||
|
|
||||||
|
for (i = 0; i < count; i++) {
|
||||||
|
m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)store[i]));
|
||||||
|
for (j = 0; j < atop(ktls_maxlen); j++) {
|
||||||
|
(void)vm_page_unwire_noq(m + j);
|
||||||
|
vm_page_free(m + j);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
ktls_free_mext_contig(struct mbuf *m)
|
||||||
|
{
|
||||||
|
M_ASSERTEXTPG(m);
|
||||||
|
uma_zfree(ktls_buffer_zone, (void *)PHYS_TO_DMAP(m->m_epg_pa[0]));
|
||||||
|
}
|
||||||
|
|
||||||
static void
|
static void
|
||||||
ktls_init(void *dummy __unused)
|
ktls_init(void *dummy __unused)
|
||||||
{
|
{
|
||||||
@ -385,6 +437,13 @@ ktls_init(void *dummy __unused)
|
|||||||
NULL, NULL, NULL, NULL,
|
NULL, NULL, NULL, NULL,
|
||||||
UMA_ALIGN_CACHE, 0);
|
UMA_ALIGN_CACHE, 0);
|
||||||
|
|
||||||
|
if (ktls_sw_buffer_cache) {
|
||||||
|
ktls_buffer_zone = uma_zcache_create("ktls_buffers",
|
||||||
|
roundup2(ktls_maxlen, PAGE_SIZE), NULL, NULL, NULL, NULL,
|
||||||
|
ktls_buffer_import, ktls_buffer_release, NULL,
|
||||||
|
UMA_ZONE_FIRSTTOUCH);
|
||||||
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Initialize the workqueues to run the TLS work. We create a
|
* Initialize the workqueues to run the TLS work. We create a
|
||||||
* work queue for each CPU.
|
* work queue for each CPU.
|
||||||
@ -1974,6 +2033,30 @@ ktls_enqueue_to_free(struct mbuf *m)
|
|||||||
wakeup(wq);
|
wakeup(wq);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void *
|
||||||
|
ktls_buffer_alloc(struct ktls_wq *wq, struct mbuf *m)
|
||||||
|
{
|
||||||
|
void *buf;
|
||||||
|
|
||||||
|
if (m->m_epg_npgs <= 2)
|
||||||
|
return (NULL);
|
||||||
|
if (ktls_buffer_zone == NULL)
|
||||||
|
return (NULL);
|
||||||
|
if ((u_int)(ticks - wq->lastallocfail) < hz) {
|
||||||
|
/*
|
||||||
|
* Rate-limit allocation attempts after a failure.
|
||||||
|
* ktls_buffer_import() will acquire a per-domain mutex to check
|
||||||
|
* the free page queues and may fail consistently if memory is
|
||||||
|
* fragmented.
|
||||||
|
*/
|
||||||
|
return (NULL);
|
||||||
|
}
|
||||||
|
buf = uma_zalloc(ktls_buffer_zone, M_NOWAIT | M_NORECLAIM);
|
||||||
|
if (buf == NULL)
|
||||||
|
wq->lastallocfail = ticks;
|
||||||
|
return (buf);
|
||||||
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
ktls_enqueue(struct mbuf *m, struct socket *so, int page_count)
|
ktls_enqueue(struct mbuf *m, struct socket *so, int page_count)
|
||||||
{
|
{
|
||||||
@ -2006,7 +2089,7 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int page_count)
|
|||||||
}
|
}
|
||||||
|
|
||||||
static __noinline void
|
static __noinline void
|
||||||
ktls_encrypt(struct mbuf *top)
|
ktls_encrypt(struct ktls_wq *wq, struct mbuf *top)
|
||||||
{
|
{
|
||||||
struct ktls_session *tls;
|
struct ktls_session *tls;
|
||||||
struct socket *so;
|
struct socket *so;
|
||||||
@ -2015,6 +2098,7 @@ ktls_encrypt(struct mbuf *top)
|
|||||||
struct iovec src_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
|
struct iovec src_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
|
||||||
struct iovec dst_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
|
struct iovec dst_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
|
||||||
vm_page_t pg;
|
vm_page_t pg;
|
||||||
|
void *cbuf;
|
||||||
int error, i, len, npages, off, total_pages;
|
int error, i, len, npages, off, total_pages;
|
||||||
bool is_anon;
|
bool is_anon;
|
||||||
|
|
||||||
@ -2056,6 +2140,9 @@ ktls_encrypt(struct mbuf *top)
|
|||||||
KASSERT(npages + m->m_epg_npgs <= total_pages,
|
KASSERT(npages + m->m_epg_npgs <= total_pages,
|
||||||
("page count mismatch: top %p, total_pages %d, m %p", top,
|
("page count mismatch: top %p, total_pages %d, m %p", top,
|
||||||
total_pages, m));
|
total_pages, m));
|
||||||
|
KASSERT(ptoa(m->m_epg_npgs) <= ktls_maxlen,
|
||||||
|
("page count %d larger than maximum frame length %d",
|
||||||
|
m->m_epg_npgs, ktls_maxlen));
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Generate source and destination ivoecs to pass to
|
* Generate source and destination ivoecs to pass to
|
||||||
@ -2072,37 +2159,50 @@ ktls_encrypt(struct mbuf *top)
|
|||||||
len = m_epg_pagelen(m, i, off);
|
len = m_epg_pagelen(m, i, off);
|
||||||
src_iov[i].iov_len = len;
|
src_iov[i].iov_len = len;
|
||||||
src_iov[i].iov_base =
|
src_iov[i].iov_base =
|
||||||
(char *)(void *)PHYS_TO_DMAP(m->m_epg_pa[i]) +
|
(char *)(void *)PHYS_TO_DMAP(m->m_epg_pa[i]) + off;
|
||||||
off;
|
}
|
||||||
|
|
||||||
if (is_anon) {
|
if (is_anon) {
|
||||||
dst_iov[i].iov_base = src_iov[i].iov_base;
|
memcpy(dst_iov, src_iov, i * sizeof(struct iovec));
|
||||||
dst_iov[i].iov_len = src_iov[i].iov_len;
|
} else if ((cbuf = ktls_buffer_alloc(wq, m)) != NULL) {
|
||||||
continue;
|
len = ptoa(m->m_epg_npgs - 1) + m->m_epg_last_len -
|
||||||
|
m->m_epg_1st_off;
|
||||||
|
dst_iov[0].iov_base = (char *)cbuf + m->m_epg_1st_off;
|
||||||
|
dst_iov[0].iov_len = len;
|
||||||
|
parray[0] = DMAP_TO_PHYS((vm_offset_t)cbuf);
|
||||||
|
i = 1;
|
||||||
|
} else {
|
||||||
|
cbuf = NULL;
|
||||||
|
off = m->m_epg_1st_off;
|
||||||
|
for (i = 0; i < m->m_epg_npgs; i++, off = 0) {
|
||||||
|
do {
|
||||||
|
pg = vm_page_alloc(NULL, 0,
|
||||||
|
VM_ALLOC_NORMAL |
|
||||||
|
VM_ALLOC_NOOBJ |
|
||||||
|
VM_ALLOC_NODUMP |
|
||||||
|
VM_ALLOC_WIRED |
|
||||||
|
VM_ALLOC_WAITFAIL);
|
||||||
|
} while (pg == NULL);
|
||||||
|
|
||||||
|
len = m_epg_pagelen(m, i, off);
|
||||||
|
parray[i] = VM_PAGE_TO_PHYS(pg);
|
||||||
|
dst_iov[i].iov_base =
|
||||||
|
(char *)(void *)PHYS_TO_DMAP(
|
||||||
|
parray[i]) + off;
|
||||||
|
dst_iov[i].iov_len = len;
|
||||||
}
|
}
|
||||||
retry_page:
|
|
||||||
pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
|
|
||||||
VM_ALLOC_NOOBJ | VM_ALLOC_NODUMP | VM_ALLOC_WIRED);
|
|
||||||
if (pg == NULL) {
|
|
||||||
vm_wait(NULL);
|
|
||||||
goto retry_page;
|
|
||||||
}
|
|
||||||
parray[i] = VM_PAGE_TO_PHYS(pg);
|
|
||||||
dst_iov[i].iov_base =
|
|
||||||
(char *)(void *)PHYS_TO_DMAP(parray[i]) + off;
|
|
||||||
dst_iov[i].iov_len = len;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (__predict_false(m->m_epg_npgs == 0)) {
|
if (__predict_false(m->m_epg_npgs == 0)) {
|
||||||
/* TLS 1.0 empty fragment. */
|
/* TLS 1.0 empty fragment. */
|
||||||
npages++;
|
npages++;
|
||||||
} else
|
} else
|
||||||
npages += i;
|
npages += m->m_epg_npgs;
|
||||||
|
|
||||||
error = (*tls->sw_encrypt)(tls,
|
error = (*tls->sw_encrypt)(tls,
|
||||||
(const struct tls_record_layer *)m->m_epg_hdr,
|
(const struct tls_record_layer *)m->m_epg_hdr,
|
||||||
m->m_epg_trail, src_iov, dst_iov, i, m->m_epg_seqno,
|
m->m_epg_trail, src_iov, dst_iov, m->m_epg_npgs, i,
|
||||||
m->m_epg_record_type);
|
m->m_epg_seqno, m->m_epg_record_type);
|
||||||
if (error) {
|
if (error) {
|
||||||
counter_u64_add(ktls_offload_failed_crypto, 1);
|
counter_u64_add(ktls_offload_failed_crypto, 1);
|
||||||
break;
|
break;
|
||||||
@ -2118,11 +2218,19 @@ ktls_encrypt(struct mbuf *top)
|
|||||||
m->m_ext.ext_free(m);
|
m->m_ext.ext_free(m);
|
||||||
|
|
||||||
/* Replace them with the new pages. */
|
/* Replace them with the new pages. */
|
||||||
for (i = 0; i < m->m_epg_npgs; i++)
|
if (cbuf != NULL) {
|
||||||
m->m_epg_pa[i] = parray[i];
|
for (i = 0; i < m->m_epg_npgs; i++)
|
||||||
|
m->m_epg_pa[i] = parray[0] + ptoa(i);
|
||||||
|
|
||||||
/* Use the basic free routine. */
|
/* Contig pages should go back to the cache. */
|
||||||
m->m_ext.ext_free = mb_free_mext_pgs;
|
m->m_ext.ext_free = ktls_free_mext_contig;
|
||||||
|
} else {
|
||||||
|
for (i = 0; i < m->m_epg_npgs; i++)
|
||||||
|
m->m_epg_pa[i] = parray[i];
|
||||||
|
|
||||||
|
/* Use the basic free routine. */
|
||||||
|
m->m_ext.ext_free = mb_free_mext_pgs;
|
||||||
|
}
|
||||||
|
|
||||||
/* Pages are now writable. */
|
/* Pages are now writable. */
|
||||||
m->m_epg_flags |= EPG_FLAG_ANON;
|
m->m_epg_flags |= EPG_FLAG_ANON;
|
||||||
@ -2189,7 +2297,7 @@ ktls_work_thread(void *ctx)
|
|||||||
ktls_free(m->m_epg_tls);
|
ktls_free(m->m_epg_tls);
|
||||||
uma_zfree(zone_mbuf, m);
|
uma_zfree(zone_mbuf, m);
|
||||||
} else {
|
} else {
|
||||||
ktls_encrypt(m);
|
ktls_encrypt(wq, m);
|
||||||
counter_u64_add(ktls_cnt_tx_queued, -1);
|
counter_u64_add(ktls_cnt_tx_queued, -1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -178,15 +178,15 @@ ktls_ocf_dispatch(struct ocf_session *os, struct cryptop *crp)
|
|||||||
static int
|
static int
|
||||||
ktls_ocf_tls_cbc_encrypt(struct ktls_session *tls,
|
ktls_ocf_tls_cbc_encrypt(struct ktls_session *tls,
|
||||||
const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
|
const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
|
||||||
struct iovec *outiov, int iovcnt, uint64_t seqno,
|
struct iovec *outiov, int iniovcnt, int outiovcnt, uint64_t seqno,
|
||||||
uint8_t record_type __unused)
|
uint8_t record_type __unused)
|
||||||
{
|
{
|
||||||
struct uio uio, out_uio;
|
struct uio uio, out_uio;
|
||||||
struct tls_mac_data ad;
|
struct tls_mac_data ad;
|
||||||
struct cryptop crp;
|
struct cryptop crp;
|
||||||
struct ocf_session *os;
|
struct ocf_session *os;
|
||||||
struct iovec iov[iovcnt + 2];
|
struct iovec iov[iniovcnt + 2];
|
||||||
struct iovec out_iov[iovcnt + 1];
|
struct iovec out_iov[outiovcnt + 1];
|
||||||
int i, error;
|
int i, error;
|
||||||
uint16_t tls_comp_len;
|
uint16_t tls_comp_len;
|
||||||
uint8_t pad;
|
uint8_t pad;
|
||||||
@ -219,10 +219,11 @@ ktls_ocf_tls_cbc_encrypt(struct ktls_session *tls,
|
|||||||
* at least compute inplace as well while we are here.
|
* at least compute inplace as well while we are here.
|
||||||
*/
|
*/
|
||||||
tls_comp_len = 0;
|
tls_comp_len = 0;
|
||||||
inplace = true;
|
inplace = iniovcnt == outiovcnt;
|
||||||
for (i = 0; i < iovcnt; i++) {
|
for (i = 0; i < iniovcnt; i++) {
|
||||||
tls_comp_len += iniov[i].iov_len;
|
tls_comp_len += iniov[i].iov_len;
|
||||||
if (iniov[i].iov_base != outiov[i].iov_base)
|
if (inplace &&
|
||||||
|
(i >= outiovcnt || iniov[i].iov_base != outiov[i].iov_base))
|
||||||
inplace = false;
|
inplace = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -236,11 +237,11 @@ ktls_ocf_tls_cbc_encrypt(struct ktls_session *tls,
|
|||||||
/* First, compute the MAC. */
|
/* First, compute the MAC. */
|
||||||
iov[0].iov_base = &ad;
|
iov[0].iov_base = &ad;
|
||||||
iov[0].iov_len = sizeof(ad);
|
iov[0].iov_len = sizeof(ad);
|
||||||
memcpy(&iov[1], iniov, sizeof(*iniov) * iovcnt);
|
memcpy(&iov[1], iniov, sizeof(*iniov) * iniovcnt);
|
||||||
iov[iovcnt + 1].iov_base = trailer;
|
iov[iniovcnt + 1].iov_base = trailer;
|
||||||
iov[iovcnt + 1].iov_len = os->mac_len;
|
iov[iniovcnt + 1].iov_len = os->mac_len;
|
||||||
uio.uio_iov = iov;
|
uio.uio_iov = iov;
|
||||||
uio.uio_iovcnt = iovcnt + 2;
|
uio.uio_iovcnt = iniovcnt + 2;
|
||||||
uio.uio_offset = 0;
|
uio.uio_offset = 0;
|
||||||
uio.uio_segflg = UIO_SYSSPACE;
|
uio.uio_segflg = UIO_SYSSPACE;
|
||||||
uio.uio_td = curthread;
|
uio.uio_td = curthread;
|
||||||
@ -279,10 +280,10 @@ ktls_ocf_tls_cbc_encrypt(struct ktls_session *tls,
|
|||||||
* Don't recopy the input iovec, instead just adjust the
|
* Don't recopy the input iovec, instead just adjust the
|
||||||
* trailer length and skip over the AAD vector in the uio.
|
* trailer length and skip over the AAD vector in the uio.
|
||||||
*/
|
*/
|
||||||
iov[iovcnt + 1].iov_len += pad + 1;
|
iov[iniovcnt + 1].iov_len += pad + 1;
|
||||||
uio.uio_iov = iov + 1;
|
uio.uio_iov = iov + 1;
|
||||||
uio.uio_iovcnt = iovcnt + 1;
|
uio.uio_iovcnt = iniovcnt + 1;
|
||||||
uio.uio_resid = tls_comp_len + iov[iovcnt + 1].iov_len;
|
uio.uio_resid = tls_comp_len + iov[iniovcnt + 1].iov_len;
|
||||||
KASSERT(uio.uio_resid % AES_BLOCK_LEN == 0,
|
KASSERT(uio.uio_resid % AES_BLOCK_LEN == 0,
|
||||||
("invalid encryption size"));
|
("invalid encryption size"));
|
||||||
|
|
||||||
@ -297,10 +298,10 @@ ktls_ocf_tls_cbc_encrypt(struct ktls_session *tls,
|
|||||||
memcpy(crp.crp_iv, hdr + 1, AES_BLOCK_LEN);
|
memcpy(crp.crp_iv, hdr + 1, AES_BLOCK_LEN);
|
||||||
crypto_use_uio(&crp, &uio);
|
crypto_use_uio(&crp, &uio);
|
||||||
if (!inplace) {
|
if (!inplace) {
|
||||||
memcpy(out_iov, outiov, sizeof(*iniov) * iovcnt);
|
memcpy(out_iov, outiov, sizeof(*iniov) * outiovcnt);
|
||||||
out_iov[iovcnt] = iov[iovcnt + 1];
|
out_iov[outiovcnt] = iov[outiovcnt + 1];
|
||||||
out_uio.uio_iov = out_iov;
|
out_uio.uio_iov = out_iov;
|
||||||
out_uio.uio_iovcnt = iovcnt + 1;
|
out_uio.uio_iovcnt = outiovcnt + 1;
|
||||||
out_uio.uio_offset = 0;
|
out_uio.uio_offset = 0;
|
||||||
out_uio.uio_segflg = UIO_SYSSPACE;
|
out_uio.uio_segflg = UIO_SYSSPACE;
|
||||||
out_uio.uio_td = curthread;
|
out_uio.uio_td = curthread;
|
||||||
@ -338,14 +339,14 @@ ktls_ocf_tls_cbc_encrypt(struct ktls_session *tls,
|
|||||||
static int
|
static int
|
||||||
ktls_ocf_tls12_aead_encrypt(struct ktls_session *tls,
|
ktls_ocf_tls12_aead_encrypt(struct ktls_session *tls,
|
||||||
const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
|
const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
|
||||||
struct iovec *outiov, int iovcnt, uint64_t seqno,
|
struct iovec *outiov, int iniovcnt, int outiovcnt, uint64_t seqno,
|
||||||
uint8_t record_type __unused)
|
uint8_t record_type __unused)
|
||||||
{
|
{
|
||||||
struct uio uio, out_uio, *tag_uio;
|
struct uio uio, out_uio, *tag_uio;
|
||||||
struct tls_aead_data ad;
|
struct tls_aead_data ad;
|
||||||
struct cryptop crp;
|
struct cryptop crp;
|
||||||
struct ocf_session *os;
|
struct ocf_session *os;
|
||||||
struct iovec iov[iovcnt + 1];
|
struct iovec iov[outiovcnt + 1];
|
||||||
int i, error;
|
int i, error;
|
||||||
uint16_t tls_comp_len;
|
uint16_t tls_comp_len;
|
||||||
bool inplace;
|
bool inplace;
|
||||||
@ -353,13 +354,13 @@ ktls_ocf_tls12_aead_encrypt(struct ktls_session *tls,
|
|||||||
os = tls->cipher;
|
os = tls->cipher;
|
||||||
|
|
||||||
uio.uio_iov = iniov;
|
uio.uio_iov = iniov;
|
||||||
uio.uio_iovcnt = iovcnt;
|
uio.uio_iovcnt = iniovcnt;
|
||||||
uio.uio_offset = 0;
|
uio.uio_offset = 0;
|
||||||
uio.uio_segflg = UIO_SYSSPACE;
|
uio.uio_segflg = UIO_SYSSPACE;
|
||||||
uio.uio_td = curthread;
|
uio.uio_td = curthread;
|
||||||
|
|
||||||
out_uio.uio_iov = outiov;
|
out_uio.uio_iov = outiov;
|
||||||
out_uio.uio_iovcnt = iovcnt;
|
out_uio.uio_iovcnt = outiovcnt;
|
||||||
out_uio.uio_offset = 0;
|
out_uio.uio_offset = 0;
|
||||||
out_uio.uio_segflg = UIO_SYSSPACE;
|
out_uio.uio_segflg = UIO_SYSSPACE;
|
||||||
out_uio.uio_td = curthread;
|
out_uio.uio_td = curthread;
|
||||||
@ -396,10 +397,11 @@ ktls_ocf_tls12_aead_encrypt(struct ktls_session *tls,
|
|||||||
crp.crp_aad_length = sizeof(ad);
|
crp.crp_aad_length = sizeof(ad);
|
||||||
|
|
||||||
/* Compute payload length and determine if encryption is in place. */
|
/* Compute payload length and determine if encryption is in place. */
|
||||||
inplace = true;
|
inplace = iniovcnt == outiovcnt;
|
||||||
crp.crp_payload_start = 0;
|
crp.crp_payload_start = 0;
|
||||||
for (i = 0; i < iovcnt; i++) {
|
for (i = 0; i < iniovcnt; i++) {
|
||||||
if (iniov[i].iov_base != outiov[i].iov_base)
|
if (inplace &&
|
||||||
|
(i >= outiovcnt || iniov[i].iov_base != outiov[i].iov_base))
|
||||||
inplace = false;
|
inplace = false;
|
||||||
crp.crp_payload_length += iniov[i].iov_len;
|
crp.crp_payload_length += iniov[i].iov_len;
|
||||||
}
|
}
|
||||||
@ -412,9 +414,9 @@ ktls_ocf_tls12_aead_encrypt(struct ktls_session *tls,
|
|||||||
tag_uio = &out_uio;
|
tag_uio = &out_uio;
|
||||||
|
|
||||||
/* Duplicate iovec and append vector for tag. */
|
/* Duplicate iovec and append vector for tag. */
|
||||||
memcpy(iov, tag_uio->uio_iov, iovcnt * sizeof(struct iovec));
|
memcpy(iov, tag_uio->uio_iov, outiovcnt * sizeof(struct iovec));
|
||||||
iov[iovcnt].iov_base = trailer;
|
iov[outiovcnt].iov_base = trailer;
|
||||||
iov[iovcnt].iov_len = AES_GMAC_HASH_LEN;
|
iov[outiovcnt].iov_len = AES_GMAC_HASH_LEN;
|
||||||
tag_uio->uio_iov = iov;
|
tag_uio->uio_iov = iov;
|
||||||
tag_uio->uio_iovcnt++;
|
tag_uio->uio_iovcnt++;
|
||||||
crp.crp_digest_start = tag_uio->uio_resid;
|
crp.crp_digest_start = tag_uio->uio_resid;
|
||||||
@ -510,14 +512,15 @@ ktls_ocf_tls12_aead_decrypt(struct ktls_session *tls,
|
|||||||
static int
|
static int
|
||||||
ktls_ocf_tls13_aead_encrypt(struct ktls_session *tls,
|
ktls_ocf_tls13_aead_encrypt(struct ktls_session *tls,
|
||||||
const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
|
const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
|
||||||
struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type)
|
struct iovec *outiov, int iniovcnt, int outiovcnt, uint64_t seqno,
|
||||||
|
uint8_t record_type)
|
||||||
{
|
{
|
||||||
struct uio uio, out_uio;
|
struct uio uio, out_uio;
|
||||||
struct tls_aead_data_13 ad;
|
struct tls_aead_data_13 ad;
|
||||||
char nonce[12];
|
char nonce[12];
|
||||||
struct cryptop crp;
|
struct cryptop crp;
|
||||||
struct ocf_session *os;
|
struct ocf_session *os;
|
||||||
struct iovec iov[iovcnt + 1], out_iov[iovcnt + 1];
|
struct iovec iov[iniovcnt + 1], out_iov[outiovcnt + 1];
|
||||||
int i, error;
|
int i, error;
|
||||||
bool inplace;
|
bool inplace;
|
||||||
|
|
||||||
@ -538,10 +541,11 @@ ktls_ocf_tls13_aead_encrypt(struct ktls_session *tls,
|
|||||||
crp.crp_aad_length = sizeof(ad);
|
crp.crp_aad_length = sizeof(ad);
|
||||||
|
|
||||||
/* Compute payload length and determine if encryption is in place. */
|
/* Compute payload length and determine if encryption is in place. */
|
||||||
inplace = true;
|
inplace = iniovcnt == outiovcnt;
|
||||||
crp.crp_payload_start = 0;
|
crp.crp_payload_start = 0;
|
||||||
for (i = 0; i < iovcnt; i++) {
|
for (i = 0; i < iniovcnt; i++) {
|
||||||
if (iniov[i].iov_base != outiov[i].iov_base)
|
if (inplace && (i >= outiovcnt ||
|
||||||
|
iniov[i].iov_base != outiov[i].iov_base))
|
||||||
inplace = false;
|
inplace = false;
|
||||||
crp.crp_payload_length += iniov[i].iov_len;
|
crp.crp_payload_length += iniov[i].iov_len;
|
||||||
}
|
}
|
||||||
@ -556,11 +560,11 @@ ktls_ocf_tls13_aead_encrypt(struct ktls_session *tls,
|
|||||||
* include the full trailer as input to get the record_type
|
* include the full trailer as input to get the record_type
|
||||||
* even if only the first byte is used.
|
* even if only the first byte is used.
|
||||||
*/
|
*/
|
||||||
memcpy(iov, iniov, iovcnt * sizeof(*iov));
|
memcpy(iov, iniov, iniovcnt * sizeof(*iov));
|
||||||
iov[iovcnt].iov_base = trailer;
|
iov[iniovcnt].iov_base = trailer;
|
||||||
iov[iovcnt].iov_len = tls->params.tls_tlen;
|
iov[iniovcnt].iov_len = tls->params.tls_tlen;
|
||||||
uio.uio_iov = iov;
|
uio.uio_iov = iov;
|
||||||
uio.uio_iovcnt = iovcnt + 1;
|
uio.uio_iovcnt = iniovcnt + 1;
|
||||||
uio.uio_offset = 0;
|
uio.uio_offset = 0;
|
||||||
uio.uio_resid = crp.crp_payload_length + tls->params.tls_tlen - 1;
|
uio.uio_resid = crp.crp_payload_length + tls->params.tls_tlen - 1;
|
||||||
uio.uio_segflg = UIO_SYSSPACE;
|
uio.uio_segflg = UIO_SYSSPACE;
|
||||||
@ -569,11 +573,11 @@ ktls_ocf_tls13_aead_encrypt(struct ktls_session *tls,
|
|||||||
|
|
||||||
if (!inplace) {
|
if (!inplace) {
|
||||||
/* Duplicate the output iov to append the trailer. */
|
/* Duplicate the output iov to append the trailer. */
|
||||||
memcpy(out_iov, outiov, iovcnt * sizeof(*out_iov));
|
memcpy(out_iov, outiov, outiovcnt * sizeof(*out_iov));
|
||||||
out_iov[iovcnt] = iov[iovcnt];
|
out_iov[outiovcnt] = iov[outiovcnt];
|
||||||
|
|
||||||
out_uio.uio_iov = out_iov;
|
out_uio.uio_iov = out_iov;
|
||||||
out_uio.uio_iovcnt = iovcnt + 1;
|
out_uio.uio_iovcnt = outiovcnt + 1;
|
||||||
out_uio.uio_offset = 0;
|
out_uio.uio_offset = 0;
|
||||||
out_uio.uio_resid = crp.crp_payload_length +
|
out_uio.uio_resid = crp.crp_payload_length +
|
||||||
tls->params.tls_tlen - 1;
|
tls->params.tls_tlen - 1;
|
||||||
|
@ -164,7 +164,7 @@ struct tls_session_params {
|
|||||||
#define KTLS_TX 1
|
#define KTLS_TX 1
|
||||||
#define KTLS_RX 2
|
#define KTLS_RX 2
|
||||||
|
|
||||||
#define KTLS_API_VERSION 7
|
#define KTLS_API_VERSION 8
|
||||||
|
|
||||||
struct iovec;
|
struct iovec;
|
||||||
struct ktls_session;
|
struct ktls_session;
|
||||||
@ -186,8 +186,8 @@ struct ktls_session {
|
|||||||
union {
|
union {
|
||||||
int (*sw_encrypt)(struct ktls_session *tls,
|
int (*sw_encrypt)(struct ktls_session *tls,
|
||||||
const struct tls_record_layer *hdr, uint8_t *trailer,
|
const struct tls_record_layer *hdr, uint8_t *trailer,
|
||||||
struct iovec *src, struct iovec *dst, int iovcnt,
|
struct iovec *src, struct iovec *dst, int srciovcnt,
|
||||||
uint64_t seqno, uint8_t record_type);
|
int dstiovcnt, uint64_t seqno, uint8_t record_type);
|
||||||
int (*sw_decrypt)(struct ktls_session *tls,
|
int (*sw_decrypt)(struct ktls_session *tls,
|
||||||
const struct tls_record_layer *hdr, struct mbuf *m,
|
const struct tls_record_layer *hdr, struct mbuf *m,
|
||||||
uint64_t seqno, int *trailer_len);
|
uint64_t seqno, int *trailer_len);
|
||||||
|
Loading…
Reference in New Issue
Block a user