d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Step 2.4: Stop using 'struct mbuf_ext_pgs' in drivers.

Browse Source 
Reviewed by:	gallatin, hselasky
Differential Revision:	https://reviews.freebsd.org/D24598
This commit is contained in:
Gleb Smirnoff 2020-05-02 23:58:20 +00:00
parent c4ee38f8e8
commit 6fbcdeb6f1
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=360576
3 changed files with 36 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
sys/dev/cxgbe/crypto/t4_kern_tls.c
View File

@ -900,12 +900,10 @@ ktls_base_wr_size(struct tlspcb *tlsp)
static u_int
ktls_tcp_payload_length(struct tlspcb *tlsp, struct mbuf *m_tls)
{
struct mbuf_ext_pgs *ext_pgs;
struct tls_record_layer *hdr;
u_int plen, mlen;
MBUF_EXT_PGS_ASSERT(m_tls);
ext_pgs = &m_tls->m_ext_pgs;
hdr = (void *)m_tls->m_epg_hdr;
plen = ntohs(hdr->tls_length);
@ -924,8 +922,8 @@ ktls_tcp_payload_length(struct tlspcb *tlsp, struct mbuf *m_tls)
* trim the length to avoid sending any of the trailer. There
* is no way to send a partial trailer currently.
*/
if (mlen > TLS_HEADER_LENGTH + plen - ext_pgs->trail_len)
mlen = TLS_HEADER_LENGTH + plen - ext_pgs->trail_len;
if (mlen > TLS_HEADER_LENGTH + plen - m_tls->m_ext_pgs.trail_len)
mlen = TLS_HEADER_LENGTH + plen - m_tls->m_ext_pgs.trail_len;
/*
@ -953,7 +951,6 @@ ktls_tcp_payload_length(struct tlspcb *tlsp, struct mbuf *m_tls)
static u_int
ktls_payload_offset(struct tlspcb *tlsp, struct mbuf *m_tls)
{
struct mbuf_ext_pgs *ext_pgs;
struct tls_record_layer *hdr;
u_int offset, plen;
#ifdef INVARIANTS
@ -961,14 +958,13 @@ ktls_payload_offset(struct tlspcb *tlsp, struct mbuf *m_tls)
#endif
MBUF_EXT_PGS_ASSERT(m_tls);
ext_pgs = &m_tls->m_ext_pgs;
hdr = (void *)m_tls->m_epg_hdr;
plen = ntohs(hdr->tls_length);
#ifdef INVARIANTS
mlen = mtod(m_tls, vm_offset_t) + m_tls->m_len;
MPASS(mlen < TLS_HEADER_LENGTH + plen);
#endif
if (mtod(m_tls, vm_offset_t) <= ext_pgs->hdr_len)
if (mtod(m_tls, vm_offset_t) <= m_tls->m_ext_pgs.hdr_len)
return (0);
if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
/*
@ -979,8 +975,8 @@ ktls_payload_offset(struct tlspcb *tlsp, struct mbuf *m_tls)
* the offset at the last byte of the record payload
* to send the last cipher block.
*/
offset = min(mtod(m_tls, vm_offset_t) - ext_pgs->hdr_len,
(plen - TLS_HEADER_LENGTH - ext_pgs->trail_len) - 1);
offset = min(mtod(m_tls, vm_offset_t) - m_tls->m_ext_pgs.hdr_len,
(plen - TLS_HEADER_LENGTH - m_tls->m_ext_pgs.trail_len) - 1);
return (rounddown(offset, AES_BLOCK_LEN));
}
return (0);
@ -1003,19 +999,17 @@ static int
ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
int *nsegsp)
{
struct mbuf_ext_pgs *ext_pgs;
struct tls_record_layer *hdr;
u_int imm_len, offset, plen, wr_len, tlen;
MBUF_EXT_PGS_ASSERT(m_tls);
ext_pgs = &m_tls->m_ext_pgs;
/*
* Determine the size of the TLS record payload to send
* excluding header and trailer.
*/
tlen = ktls_tcp_payload_length(tlsp, m_tls);
if (tlen <= ext_pgs->hdr_len) {
if (tlen <= m_tls->m_ext_pgs.hdr_len) {
/*
* For requests that only want to send the TLS header,
* send a tunnelled packet as immediate data.
@ -1041,7 +1035,7 @@ ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
}
hdr = (void *)m_tls->m_epg_hdr;
plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len;
plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - m_tls->m_ext_pgs.trail_len;
if (tlen < plen) {
plen = tlen;
offset = ktls_payload_offset(tlsp, m_tls);
@ -1058,14 +1052,14 @@ ktls_wr_len(struct tlspcb *tlsp, struct mbuf *m, struct mbuf *m_tls,
*/
imm_len = 0;
if (offset == 0)
imm_len += ext_pgs->hdr_len;
imm_len += m_tls->m_ext_pgs.hdr_len;
if (plen == tlen)
imm_len += AES_BLOCK_LEN;
wr_len += roundup2(imm_len, 16);
/* TLS record payload via DSGL. */
*nsegsp = sglist_count_mbuf_epg(m_tls, ext_pgs->hdr_len + offset,
plen - (ext_pgs->hdr_len + offset));
*nsegsp = sglist_count_mbuf_epg(m_tls, m_tls->m_ext_pgs.hdr_len + offset,
plen - (m_tls->m_ext_pgs.hdr_len + offset));
wr_len += ktls_sgl_size(*nsegsp);
wr_len = roundup2(wr_len, 16);
@ -1466,7 +1460,6 @@ ktls_write_tunnel_packet(struct sge_txq *txq, void *dst, struct mbuf *m,
struct ip *ip, newip;
struct ip6_hdr *ip6, newip6;
struct tcphdr *tcp, newtcp;
struct mbuf_ext_pgs *ext_pgs;
caddr_t out;
TXQ_LOCK_ASSERT_OWNED(txq);
@ -1474,7 +1467,6 @@ ktls_write_tunnel_packet(struct sge_txq *txq, void *dst, struct mbuf *m,
/* Locate the template TLS header. */
MBUF_EXT_PGS_ASSERT(m_tls);
ext_pgs = &m_tls->m_ext_pgs;
/* This should always be the last TLS record in a chain. */
MPASS(m_tls->m_next == NULL);
@ -1577,7 +1569,6 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
struct ulptx_idata *idata;
struct cpl_tx_sec_pdu *sec_pdu;
struct cpl_tx_data *tx_data;
struct mbuf_ext_pgs *ext_pgs;
struct tls_record_layer *hdr;
char *iv, *out;
u_int aad_start, aad_stop;
@ -1603,20 +1594,19 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
/* Locate the TLS header. */
MBUF_EXT_PGS_ASSERT(m_tls);
ext_pgs = &m_tls->m_ext_pgs;
hdr = (void *)m_tls->m_epg_hdr;
plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - ext_pgs->trail_len;
plen = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) - m_tls->m_ext_pgs.trail_len;
/* Determine how much of the TLS record to send. */
tlen = ktls_tcp_payload_length(tlsp, m_tls);
if (tlen <= ext_pgs->hdr_len) {
if (tlen <= m_tls->m_ext_pgs.hdr_len) {
/*
* For requests that only want to send the TLS header,
* send a tunnelled packet as immediate data.
*/
#ifdef VERBOSE_TRACES
CTR3(KTR_CXGBE, "%s: tid %d header-only TLS record %u",
__func__, tlsp->tid, (u_int)ext_pgs->seqno);
__func__, tlsp->tid, (u_int)m_tls->m_ext_pgs.seqno);
#endif
return (ktls_write_tunnel_packet(txq, dst, m, m_tls, available,
tcp_seqno, pidx));
@ -1626,7 +1616,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
offset = ktls_payload_offset(tlsp, m_tls);
#ifdef VERBOSE_TRACES
CTR4(KTR_CXGBE, "%s: tid %d short TLS record %u with offset %u",
__func__, tlsp->tid, (u_int)ext_pgs->seqno, offset);
__func__, tlsp->tid, (u_int)m_tls->m_ext_pgs.seqno, offset);
#endif
if (m_tls->m_next == NULL && (tcp->th_flags & TH_FIN) != 0) {
txq->kern_tls_fin_short++;
@ -1681,10 +1671,10 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
*/
tx_max_offset = mtod(m_tls, vm_offset_t);
if (tx_max_offset > TLS_HEADER_LENGTH + ntohs(hdr->tls_length) -
ext_pgs->trail_len) {
m_tls->m_ext_pgs.trail_len) {
/* Always send the full trailer. */
tx_max_offset = TLS_HEADER_LENGTH + ntohs(hdr->tls_length) -
ext_pgs->trail_len;
m_tls->m_ext_pgs.trail_len;
}
if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_CBC &&
tx_max_offset > TLS_HEADER_LENGTH) {
@ -1799,15 +1789,15 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
/* Recalculate 'nsegs' if cached value is not available. */
if (nsegs == 0)
nsegs = sglist_count_mbuf_epg(m_tls, ext_pgs->hdr_len +
offset, plen - (ext_pgs->hdr_len + offset));
nsegs = sglist_count_mbuf_epg(m_tls, m_tls->m_ext_pgs.hdr_len +
offset, plen - (m_tls->m_ext_pgs.hdr_len + offset));
/* Calculate the size of the TLS work request. */
twr_len = ktls_base_wr_size(tlsp);
imm_len = 0;
if (offset == 0)
imm_len += ext_pgs->hdr_len;
imm_len += m_tls->m_ext_pgs.hdr_len;
if (plen == tlen)
imm_len += AES_BLOCK_LEN;
twr_len += roundup2(imm_len, 16);
@ -1923,13 +1913,13 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
cipher_stop = 0;
sec_pdu->pldlen = htobe32(16 + plen -
(ext_pgs->hdr_len + offset));
(m_tls->m_ext_pgs.hdr_len + offset));
/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
sec_pdu->seqno_numivs = tlsp->scmd0_short.seqno_numivs;
sec_pdu->ivgen_hdrlen = htobe32(
tlsp->scmd0_short.ivgen_hdrlen |
V_SCMD_HDR_LEN(offset == 0 ? ext_pgs->hdr_len : 0));
V_SCMD_HDR_LEN(offset == 0 ? m_tls->m_ext_pgs.hdr_len : 0));
txq->kern_tls_short++;
} else {
@ -1942,7 +1932,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
aad_start = 1;
aad_stop = TLS_HEADER_LENGTH;
iv_offset = TLS_HEADER_LENGTH + 1;
cipher_start = ext_pgs->hdr_len + 1;
cipher_start = m_tls->m_ext_pgs.hdr_len + 1;
if (tlsp->enc_mode == SCMD_CIPH_MODE_AES_GCM) {
cipher_stop = 0;
auth_start = cipher_start;
@ -1981,7 +1971,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
V_CPL_TX_SEC_PDU_AUTHSTOP(auth_stop) |
V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert));
sec_pdu->scmd1 = htobe64(ext_pgs->seqno);
sec_pdu->scmd1 = htobe64(m_tls->m_ext_pgs.seqno);
/* Key context */
out = (void *)(sec_pdu + 1);
@ -2021,8 +2011,8 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
tx_data->rsvd = htobe32(tcp_seqno);
} else {
tx_data->len = htobe32(V_TX_DATA_MSS(mss) |
V_TX_LENGTH(tlen - (ext_pgs->hdr_len + offset)));
tx_data->rsvd = htobe32(tcp_seqno + ext_pgs->hdr_len + offset);
V_TX_LENGTH(tlen - (m_tls->m_ext_pgs.hdr_len + offset)));
tx_data->rsvd = htobe32(tcp_seqno + m_tls->m_ext_pgs.hdr_len + offset);
}
tx_data->flags = htobe32(F_TX_BYPASS);
if (last_wr && tcp->th_flags & TH_PUSH)
@ -2031,8 +2021,8 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
/* Populate the TLS header */
out = (void *)(tx_data + 1);
if (offset == 0) {
memcpy(out, m_tls->m_epg_hdr, ext_pgs->hdr_len);
out += ext_pgs->hdr_len;
memcpy(out, m_tls->m_epg_hdr, m_tls->m_ext_pgs.hdr_len);
out += m_tls->m_ext_pgs.hdr_len;
}
/* AES IV for a short record. */
@ -2067,8 +2057,8 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
/* SGL for record payload */
sglist_reset(txq->gl);
if (sglist_append_mbuf_epg(txq->gl, m_tls, ext_pgs->hdr_len + offset,
plen - (ext_pgs->hdr_len + offset)) != 0) {
if (sglist_append_mbuf_epg(txq->gl, m_tls, m_tls->m_ext_pgs.hdr_len + offset,
plen - (m_tls->m_ext_pgs.hdr_len + offset)) != 0) {
#ifdef INVARIANTS
panic("%s: failed to append sglist", __func__);
#endif
@ -2090,7 +2080,7 @@ ktls_write_tls_wr(struct tlspcb *tlsp, struct sge_txq *txq,
txq->kern_tls_waste += mtod(m_tls, vm_offset_t);
else
txq->kern_tls_waste += mtod(m_tls, vm_offset_t) -
(ext_pgs->hdr_len + offset);
(m_tls->m_ext_pgs.hdr_len + offset);
}
txsd = &txq->sdesc[pidx];

14
sys/dev/cxgbe/tom/t4_cpl_io.c
View File

@ -1924,19 +1924,17 @@ aiotx_free_job(struct kaiocb *job)
static void
aiotx_free_pgs(struct mbuf *m)
{
struct mbuf_ext_pgs *ext_pgs;
struct kaiocb *job;
vm_page_t pg;
MBUF_EXT_PGS_ASSERT(m);
ext_pgs = &m->m_ext_pgs;
job = m->m_ext.ext_arg1;
#ifdef VERBOSE_TRACES
CTR3(KTR_CXGBE, "%s: completed %d bytes for tid %d", __func__,
m->m_len, jobtotid(job));
#endif
for (int i = 0; i < ext_pgs->npgs; i++) {
for (int i = 0; i < m->m_ext_pgs.npgs; i++) {
pg = PHYS_TO_VM_PAGE(m->m_epg_pa[i]);
vm_page_unwire(pg, PQ_ACTIVE);
}
@ -1954,7 +1952,6 @@ alloc_aiotx_mbuf(struct kaiocb *job, int len)
struct vmspace *vm;
vm_page_t pgs[MBUF_PEXT_MAX_PGS];
struct mbuf *m, *top, *last;
struct mbuf_ext_pgs *ext_pgs;
vm_map_t map;
vm_offset_t start;
int i, mlen, npages, pgoff;
@ -1992,16 +1989,15 @@ alloc_aiotx_mbuf(struct kaiocb *job, int len)
break;
}
ext_pgs = &m->m_ext_pgs;
ext_pgs->first_pg_off = pgoff;
ext_pgs->npgs = npages;
m->m_ext_pgs.first_pg_off = pgoff;
m->m_ext_pgs.npgs = npages;
if (npages == 1) {
KASSERT(mlen + pgoff <= PAGE_SIZE,
("%s: single page is too large (off %d len %d)",
__func__, pgoff, mlen));
ext_pgs->last_pg_len = mlen;
m->m_ext_pgs.last_pg_len = mlen;
} else {
ext_pgs->last_pg_len = mlen - (PAGE_SIZE - pgoff) -
m->m_ext_pgs.last_pg_len = mlen - (PAGE_SIZE - pgoff) -
(npages - 2) * PAGE_SIZE;
}
for (i = 0; i < npages; i++)

4
sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c
View File

@ -680,13 +680,11 @@ sbtls_recover_record(struct mbuf *mb, int wait, uint32_t tcp_old, uint32_t *ptcp
static int
mlx5e_sq_tls_populate(struct mbuf *mb, uint64_t *pseq)
{
struct mbuf_ext_pgs *ext_pgs;
for (; mb != NULL; mb = mb->m_next) {
if (!(mb->m_flags & M_NOMAP))
continue;
ext_pgs = &mb->m_ext_pgs;
*pseq = ext_pgs->seqno;
*pseq = mb->m_ext_pgs.seqno;
return (1);
}
return (0);