d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Step 2.1: Build TLS workqueue from mbufs, not struct mbuf_ext_pgs.

Browse Source 
Reviewed by:	gallatin
Differential Revision:	https://reviews.freebsd.org/D24598
This commit is contained in:
Gleb Smirnoff 2020-05-02 23:38:13 +00:00
parent eeec834855
commit d90fe9d0cd
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=360573
4 changed files with 26 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
sys/kern/kern_mbuf.c
View File

@ -1246,7 +1246,6 @@ mb_free_ext(struct mbuf *m)
break;
case EXT_PGS: {
#ifdef KERN_TLS
struct mbuf_ext_pgs *pgs;
struct ktls_session *tls;
#endif
@ -1254,11 +1253,10 @@ mb_free_ext(struct mbuf *m)
("%s: ext_free not set", __func__));
mref->m_ext.ext_free(mref);
#ifdef KERN_TLS
pgs = &mref->m_ext_pgs;
tls = pgs->tls;
tls = mref->m_ext_pgs.tls;
if (tls != NULL &&
!refcount_release_if_not_last(&tls->refcount))
ktls_enqueue_to_free(pgs);
ktls_enqueue_to_free(mref);
else
#endif
uma_zfree(zone_mbuf, mref);

45
sys/kern/uipc_ktls.c
View File

@ -79,7 +79,7 @@ __FBSDID("$FreeBSD$");
struct ktls_wq {
struct mtx mtx;
STAILQ_HEAD(, mbuf_ext_pgs) head;
STAILQ_HEAD(, mbuf) head;
bool running;
} __aligned(CACHE_LINE_SIZE);
@ -1430,16 +1430,19 @@ ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
}
void
ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs)
ktls_enqueue_to_free(struct mbuf *m)
{
struct mbuf_ext_pgs *pgs;
struct ktls_wq *wq;
bool running;
pgs = &m->m_ext_pgs;
/* Mark it for freeing. */
pgs->flags |= EPG_FLAG_2FREE;
wq = &ktls_wq[pgs->tls->wq_index];
mtx_lock(&wq->mtx);
STAILQ_INSERT_TAIL(&wq->head, pgs, stailq);
STAILQ_INSERT_TAIL(&wq->head, m, m_ext_pgs.stailq);
running = wq->running;
mtx_unlock(&wq->mtx);
if (!running)
@ -1472,7 +1475,7 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int page_count)
wq = &ktls_wq[pgs->tls->wq_index];
mtx_lock(&wq->mtx);
STAILQ_INSERT_TAIL(&wq->head, pgs, stailq);
STAILQ_INSERT_TAIL(&wq->head, m, m_ext_pgs.stailq);
running = wq->running;
mtx_unlock(&wq->mtx);
if (!running)
@ -1481,11 +1484,12 @@ ktls_enqueue(struct mbuf *m, struct socket *so, int page_count)
}
static __noinline void
ktls_encrypt(struct mbuf_ext_pgs *pgs)
ktls_encrypt(struct mbuf *top)
{
struct ktls_session *tls;
struct socket *so;
struct mbuf *m, *top;
struct mbuf *m;
struct mbuf_ext_pgs *pgs;
vm_paddr_t parray[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
struct iovec src_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
struct iovec dst_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
@ -1493,15 +1497,14 @@ ktls_encrypt(struct mbuf_ext_pgs *pgs)
int error, i, len, npages, off, total_pages;
bool is_anon;
so = pgs->so;
tls = pgs->tls;
top = __containerof(pgs, struct mbuf, m_ext_pgs);
KASSERT(tls != NULL, ("tls = NULL, top = %p, pgs = %p\n", top, pgs));
KASSERT(so != NULL, ("so = NULL, top = %p, pgs = %p\n", top, pgs));
so = top->m_ext_pgs.so;
tls = top->m_ext_pgs.tls;
KASSERT(tls != NULL, ("tls = NULL, top = %p\n", top));
KASSERT(so != NULL, ("so = NULL, top = %p\n", top));
#ifdef INVARIANTS
pgs->so = NULL;
top->m_ext_pgs.so = NULL;
#endif
total_pages = pgs->enc_cnt;
total_pages = top->m_ext_pgs.enc_cnt;
npages = 0;
/*
@ -1631,10 +1634,8 @@ static void
ktls_work_thread(void *ctx)
{
struct ktls_wq *wq = ctx;
struct mbuf_ext_pgs *p, *n;
struct ktls_session *tls;
struct mbuf *m;
STAILQ_HEAD(, mbuf_ext_pgs) local_head;
struct mbuf *m, *n;
STAILQ_HEAD(, mbuf) local_head;
#if defined(__aarch64__) || defined(__amd64__) || defined(__i386__)
fpu_kern_thread(0);
@ -1651,14 +1652,12 @@ ktls_work_thread(void *ctx)
STAILQ_CONCAT(&local_head, &wq->head);
mtx_unlock(&wq->mtx);
STAILQ_FOREACH_SAFE(p, &local_head, stailq, n) {
if (p->flags & EPG_FLAG_2FREE) {
tls = p->tls;
ktls_free(tls);
m = __containerof(p, struct mbuf, m_ext_pgs);
STAILQ_FOREACH_SAFE(m, &local_head, m_ext_pgs.stailq, n) {
if (m->m_ext_pgs.flags & EPG_FLAG_2FREE) {
ktls_free(m->m_ext_pgs.tls);
uma_zfree(zone_mbuf, m);
} else {
ktls_encrypt(p);
ktls_encrypt(m);
counter_u64_add(ktls_cnt_on, -1);
}
}

3
sys/sys/ktls.h
View File

@ -169,7 +169,6 @@ struct iovec;
struct ktls_session;
struct m_snd_tag;
struct mbuf;
struct mbuf_ext_pgs;
struct sockbuf;
struct socket;
@ -212,7 +211,7 @@ void ktls_frame(struct mbuf *m, struct ktls_session *tls, int *enqueue_cnt,
uint8_t record_type);
void ktls_seq(struct sockbuf *sb, struct mbuf *m);
void ktls_enqueue(struct mbuf *m, struct socket *so, int page_count);
void ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs);
void ktls_enqueue_to_free(struct mbuf *m);
int ktls_get_rx_mode(struct socket *so);
int ktls_set_tx_mode(struct socket *so, int mode);
int ktls_get_tx_mode(struct socket *so);

2
sys/sys/mbuf.h
View File

@ -372,7 +372,7 @@ struct mbuf {
struct ktls_session *tls;
struct socket *so;
uint64_t seqno;
STAILQ_ENTRY(mbuf_ext_pgs) stailq;
STAILQ_ENTRY(mbuf) stailq;
} m_ext_pgs;
};
union {