d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

crypto/cnxk: improve fast path

Browse Source 
Remove SA & packet accesses in dequeue path by adjusting the headers in
the enqueue path for outbound packets. For inbound packets, add extra
esn_en flag in the SA to minimize cache line accesses in the datapath.

Also, use seq_lo for IPID. IPID just need to be unique. Instead of
incrementing per packet, use ESN low bits.

Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Acked-by: Akhil Goyal <gakhil@marvell.com>
This commit is contained in:
Anoob Joseph 2022-06-20 12:48:06 +05:30 committed by Akhil Goyal
parent 532963b807
commit 4c30c94a00
5 changed files with 87 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
drivers/crypto/cnxk/cn9k_cryptodev_ops.c
View File

@ -43,10 +43,12 @@ cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op,
struct cpt_inst_s *inst)
{
struct rte_crypto_sym_op *sym_op = op->sym;
struct roc_ie_on_common_sa *common_sa;
struct cn9k_sec_session *priv;
struct roc_ie_on_sa_ctl *ctl;
struct cn9k_ipsec_sa *sa;
int ret;
priv = get_sec_session_private_data(op->sym->sec_session);
sa = &priv->sa;
if (unlikely(sym_op->m_dst && sym_op->m_dst != sym_op->m_src)) {
plt_dp_err("Out of place is not supported");
@ -58,21 +60,17 @@ cn9k_cpt_sec_inst_fill(struct rte_crypto_op *op,
return -ENOTSUP;
}
priv = get_sec_session_private_data(op->sym->sec_session);
sa = &priv->sa;
if (sa->dir == RTE_SECURITY_IPSEC_SA_DIR_EGRESS)
return process_outb_sa(op, sa, inst);
ret = process_outb_sa(op, sa, inst);
else {
infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND;
process_inb_sa(op, sa, inst);
if (unlikely(sa->esn_en))
infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_INB_ESN;
ret = 0;
}
infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_DIR_INBOUND;
common_sa = &sa->in_sa.common_sa;
ctl = &common_sa->ctl;
if (ctl->esn_en)
infl_req->op_flags |= CPT_OP_FLAGS_IPSEC_INB_ESN;
return process_inb_sa(op, sa, inst);
return ret;
}
static inline struct cnxk_se_sess *
@ -234,19 +232,29 @@ cn9k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
};
pend_q = &qp->pend_q;
const uint64_t lmt_base = qp->lf.lmt_base;
const uint64_t io_addr = qp->lf.io_addr;
const uint64_t pq_mask = pend_q->pq_mask;
rte_prefetch2(pend_q);
/* Clear w0, w2, w3 of both inst */
#if defined(RTE_ARCH_ARM64)
uint64x2_t zero = vdupq_n_u64(0);
vst1q_u64(&inst[0].w0.u64, zero);
vst1q_u64(&inst[1].w0.u64, zero);
vst1q_u64(&inst[0].w2.u64, zero);
vst1q_u64(&inst[1].w2.u64, zero);
#else
inst[0].w0.u64 = 0;
inst[0].w2.u64 = 0;
inst[0].w3.u64 = 0;
inst[1].w0.u64 = 0;
inst[1].w2.u64 = 0;
inst[1].w3.u64 = 0;
#endif
const uint64_t lmt_base = qp->lf.lmt_base;
const uint64_t io_addr = qp->lf.io_addr;
const uint64_t pq_mask = pend_q->pq_mask;
head = pend_q->head;
nb_allowed = pending_queue_free_cnt(head, pend_q->tail, pq_mask);
@ -506,21 +514,26 @@ cn9k_cpt_sec_post_process(struct rte_crypto_op *cop,
uint16_t m_len = 0;
char *data;
priv = get_sec_session_private_data(cop->sym->sec_session);
sa = &priv->sa;
if (infl_req->op_flags & CPT_OP_FLAGS_IPSEC_DIR_INBOUND) {
struct roc_ie_on_common_sa *common_sa = &sa->in_sa.common_sa;
struct roc_ie_on_common_sa *common_sa;
data = rte_pktmbuf_mtod(m, char *);
if (infl_req->op_flags == CPT_OP_FLAGS_IPSEC_INB_ESN) {
struct roc_ie_on_inb_hdr *inb_hdr =
(struct roc_ie_on_inb_hdr *)data;
uint64_t seq = rte_be_to_cpu_64(inb_hdr->seq);
if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_IPSEC_INB_ESN)) {
struct roc_ie_on_inb_hdr *inb_hdr;
uint64_t seq;
priv = get_sec_session_private_data(
sym_op->sec_session);
sa = &priv->sa;
common_sa = &sa->in_sa.common_sa;
inb_hdr = (struct roc_ie_on_inb_hdr *)data;
seq = rte_be_to_cpu_64(inb_hdr->seq);
if (seq > common_sa->seq_t.u64)
common_sa->seq_t.u64 = seq;
}
ip = (struct rte_ipv4_hdr *)(data + ROC_IE_ON_INB_RPTR_HDR);
if (((ip->version_ihl & 0xf0) >> RTE_IPV4_IHL_MULTIPLIER) ==
@ -537,8 +550,6 @@ cn9k_cpt_sec_post_process(struct rte_crypto_op *cop,
m->data_len = m_len;
m->pkt_len = m_len;
m->data_off += ROC_IE_ON_INB_RPTR_HDR;
} else {
rte_pktmbuf_adj(m, sa->custom_hdr_len);
}
}

11
drivers/crypto/cnxk/cn9k_ipsec.c
View File

@ -40,13 +40,8 @@ cn9k_ipsec_outb_sa_create(struct cnxk_cpt_qp *qp,
/* Initialize lookaside IPsec private data */
sa->dir = RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
/* Start ip id from 1 */
sa->ip_id = 1;
sa->seq_lo = 1;
sa->seq_hi = 0;
if (ipsec->esn.value)
sa->esn = ipsec->esn.value;
sa->esn = ipsec->esn.value;
ret = cnxk_ipsec_outb_rlens_get(&sa->rlens, ipsec, crypto_xform);
if (ret)
@ -166,10 +161,12 @@ cn9k_ipsec_inb_sa_create(struct cnxk_cpt_qp *qp,
}
ret = cnxk_on_ipsec_inb_sa_create(ipsec, crypto_xform, &sa->in_sa);
if (ret < 0)
return ret;
if (sa->in_sa.common_sa.ctl.esn_en)
sa->esn_en = 1;
ctx_len = ret;
opcode = ROC_IE_ON_MAJOR_OP_WRITE_IPSEC_INBOUND;
egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE];

7
drivers/crypto/cnxk/cn9k_ipsec.h
View File

@ -28,8 +28,6 @@ struct cn9k_ipsec_sa {
uint8_t custom_hdr_len;
/** Response length calculation data */
struct cnxk_ipsec_outb_rlens rlens;
/** Outbound IP-ID */
uint16_t ip_id;
/** ESN */
union {
uint64_t esn;
@ -42,6 +40,11 @@ struct cn9k_ipsec_sa {
struct cnxk_on_ipsec_ar ar;
/** Anti replay window size */
uint32_t replay_win_sz;
/*
* ESN enable flag. Copy of in_sa ctl.esn_en to have single cache line
* access in the non-esn fastpath.
*/
uint8_t esn_en;
/** Queue pair */
struct cnxk_cpt_qp *qp;
};

55
drivers/crypto/cnxk/cn9k_ipsec_la_ops.h
View File

@ -77,29 +77,36 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_ipsec_sa *sa,
const unsigned int hdr_len = sa->custom_hdr_len;
struct rte_crypto_sym_op *sym_op = cop->sym;
struct rte_mbuf *m_src = sym_op->m_src;
uint32_t dlen, rlen, pkt_len, seq_lo;
uint16_t data_off = m_src->data_off;
struct roc_ie_on_outb_hdr *hdr;
uint32_t dlen, rlen;
int32_t extend_tail;
uint64_t esn;
dlen = rte_pktmbuf_pkt_len(m_src) + hdr_len;
rlen = ipsec_po_out_rlen_get(sa, dlen - hdr_len);
pkt_len = rte_pktmbuf_pkt_len(m_src);
dlen = pkt_len + hdr_len;
rlen = ipsec_po_out_rlen_get(sa, pkt_len);
extend_tail = rlen - dlen;
if (unlikely(extend_tail > rte_pktmbuf_tailroom(m_src))) {
plt_dp_err("Not enough tail room (required: %d, available: %d",
plt_dp_err("Not enough tail room (required: %d, available: %d)",
extend_tail, rte_pktmbuf_tailroom(m_src));
return -ENOMEM;
}
m_src->data_len += extend_tail;
m_src->pkt_len += extend_tail;
hdr = (struct roc_ie_on_outb_hdr *)rte_pktmbuf_prepend(m_src, hdr_len);
if (unlikely(hdr == NULL)) {
plt_dp_err("Not enough head room");
if (unlikely(hdr_len > data_off)) {
plt_dp_err("Not enough head room (required: %d, available: %d)",
hdr_len, rte_pktmbuf_headroom(m_src));
return -ENOMEM;
}
pkt_len += extend_tail;
m_src->data_len = pkt_len;
m_src->pkt_len = pkt_len;
hdr = PLT_PTR_ADD(m_src->buf_addr, data_off - hdr_len);
#ifdef LA_IPSEC_DEBUG
if (sa->inst.w4 & ROC_IE_ON_PER_PKT_IV) {
memcpy(&hdr->iv[0],
@ -109,23 +116,28 @@ process_outb_sa(struct rte_crypto_op *cop, struct cn9k_ipsec_sa *sa,
}
#endif
hdr->seq = rte_cpu_to_be_32(sa->seq_lo);
hdr->ip_id = rte_cpu_to_be_32(sa->ip_id);
hdr->esn = rte_cpu_to_be_32(sa->seq_hi);
esn = ++sa->esn;
sa->ip_id++;
sa->esn++;
/* Set ESN seq hi */
hdr->esn = rte_cpu_to_be_32(esn >> 32);
/* Set ESN seq lo */
seq_lo = rte_cpu_to_be_32(esn & (BIT_ULL(32) - 1));
hdr->seq = seq_lo;
/* Set IPID same as seq_lo */
hdr->ip_id = seq_lo;
/* Prepare CPT instruction */
inst->w4.u64 = sa->inst.w4 | dlen;
inst->dptr = rte_pktmbuf_iova(m_src);
inst->rptr = inst->dptr;
inst->dptr = PLT_U64_CAST(hdr);
inst->rptr = PLT_U64_CAST(hdr);
inst->w7.u64 = sa->inst.w7;
return 0;
}
static __rte_always_inline int
static __rte_always_inline void
process_inb_sa(struct rte_crypto_op *cop, struct cn9k_ipsec_sa *sa,
struct cpt_inst_s *inst)
{
@ -149,16 +161,13 @@ process_inb_sa(struct rte_crypto_op *cop, struct cn9k_ipsec_sa *sa,
inst->dptr = rte_pktmbuf_iova(m_src);
inst->rptr = inst->dptr;
inst->w7.u64 = sa->inst.w7;
return 0;
return;
}
}
/* Prepare CPT instruction */
inst->w4.u64 = sa->inst.w4 | rte_pktmbuf_pkt_len(m_src);
inst->dptr = rte_pktmbuf_iova(m_src);
inst->rptr = inst->dptr;
inst->dptr = inst->rptr = rte_pktmbuf_iova(m_src);
inst->w7.u64 = sa->inst.w7;
return 0;
}
#endif /* __CN9K_IPSEC_LA_OPS_H__ */

12
drivers/crypto/cnxk/cnxk_cryptodev_ops.h
View File

@ -70,16 +70,16 @@ struct cnxk_cpt_qp {
/**< Crypto LF */
struct pending_queue pend_q;
/**< Pending queue */
struct roc_cpt_lmtline lmtline;
/**< Lmtline information */
struct cpt_qp_meta_info meta_info;
/**< Metabuf info required to support operations on the queue pair */
struct crypto_adpter_info ca;
/**< Crypto adapter related info */
struct rte_mempool *sess_mp;
/**< Session mempool */
struct rte_mempool *sess_mp_priv;
/**< Session private data mempool */
struct cpt_qp_meta_info meta_info;
/**< Metabuf info required to support operations on the queue pair */
struct roc_cpt_lmtline lmtline;
/**< Lmtline information */
struct crypto_adpter_info ca;
/**< Crypto adapter related info */
};
int cnxk_cpt_dev_config(struct rte_cryptodev *dev,