d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

crypto/cnxk: add enqueue burst

Browse Source 
Add enqueue_burst op in cn9k & cn10k.

Signed-off-by: Ankur Dwivedi <adwivedi@marvell.com>
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Signed-off-by: Archana Muniganti <marchana@marvell.com>
Signed-off-by: Tejasree Kondoj <ktejasree@marvell.com>
Acked-by: Akhil Goyal <gakhil@marvell.com>
This commit is contained in:
Anoob Joseph 2021-06-25 11:26:17 +05:30 committed by Akhil Goyal
parent 7e0189607c
commit 75ee574573
7 changed files with 360 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/crypto/cnxk/cn10k_cryptodev.c
View File

@ -80,6 +80,8 @@ cn10k_cpt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
dev->dev_ops = &cn10k_cpt_ops;
dev->driver_id = cn10k_cryptodev_driver_id;
cn10k_cpt_set_enqdeq_fns(dev);
return 0;
dev_fini:

189
drivers/crypto/cnxk/cn10k_cryptodev_ops.c
View File

@ -7,7 +7,196 @@
#include "cn10k_cryptodev.h"
#include "cn10k_cryptodev_ops.h"
#include "cnxk_cryptodev.h"
#include "cnxk_cryptodev_ops.h"
#include "cnxk_se.h"
static inline struct cnxk_se_sess *
cn10k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op)
{
const int driver_id = cn10k_cryptodev_driver_id;
struct rte_crypto_sym_op *sym_op = op->sym;
struct rte_cryptodev_sym_session *sess;
struct cnxk_se_sess *priv;
int ret;
/* Create temporary session */
sess = rte_cryptodev_sym_session_create(qp->sess_mp);
if (sess == NULL)
return NULL;
ret = sym_session_configure(qp->lf.roc_cpt, driver_id, sym_op->xform,
sess, qp->sess_mp_priv);
if (ret)
goto sess_put;
priv = get_sym_session_private_data(sess, driver_id);
sym_op->session = sess;
return priv;
sess_put:
rte_mempool_put(qp->sess_mp, sess);
return NULL;
}
static __rte_always_inline int __rte_hot
cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
struct cnxk_se_sess *sess, struct cpt_inflight_req *infl_req,
struct cpt_inst_s *inst)
{
RTE_SET_USED(qp);
RTE_SET_USED(op);
RTE_SET_USED(sess);
RTE_SET_USED(infl_req);
RTE_SET_USED(inst);
return -ENOTSUP;
}
static inline int
cn10k_cpt_fill_inst(struct cnxk_cpt_qp *qp, struct rte_crypto_op *ops[],
struct cpt_inst_s inst[], struct cpt_inflight_req *infl_req)
{
struct rte_crypto_sym_op *sym_op;
struct cnxk_se_sess *sess;
struct rte_crypto_op *op;
uint64_t w7;
int ret;
op = ops[0];
inst[0].w0.u64 = 0;
inst[0].w2.u64 = 0;
inst[0].w3.u64 = 0;
sym_op = op->sym;
if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
sess = get_sym_session_private_data(
sym_op->session, cn10k_cryptodev_driver_id);
ret = cpt_sym_inst_fill(qp, op, sess, infl_req,
&inst[0]);
if (unlikely(ret))
return 0;
w7 = sess->cpt_inst_w7;
} else {
sess = cn10k_cpt_sym_temp_sess_create(qp, op);
if (unlikely(sess == NULL)) {
plt_dp_err("Could not create temp session");
return 0;
}
ret = cpt_sym_inst_fill(qp, op, sess, infl_req,
&inst[0]);
if (unlikely(ret)) {
sym_session_clear(cn10k_cryptodev_driver_id,
op->sym->session);
rte_mempool_put(qp->sess_mp, op->sym->session);
return 0;
}
w7 = sess->cpt_inst_w7;
}
} else {
plt_dp_err("Unsupported op type");
return 0;
}
inst[0].res_addr = (uint64_t)&infl_req->res;
infl_req->res.cn10k.compcode = CPT_COMP_NOT_DONE;
infl_req->cop = op;
inst[0].w7.u64 = w7;
return 1;
}
#define PKTS_PER_LOOP 32
#define PKTS_PER_STEORL 16
static uint16_t
cn10k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
{
uint64_t lmt_base, lmt_arg, io_addr;
struct cpt_inflight_req *infl_req;
uint16_t nb_allowed, count = 0;
struct cnxk_cpt_qp *qp = qptr;
struct pending_queue *pend_q;
struct cpt_inst_s *inst;
uint16_t lmt_id;
int ret, i;
pend_q = &qp->pend_q;
nb_allowed = qp->lf.nb_desc - pend_q->pending_count;
nb_ops = RTE_MIN(nb_ops, nb_allowed);
if (unlikely(nb_ops == 0))
return 0;
lmt_base = qp->lmtline.lmt_base;
io_addr = qp->lmtline.io_addr;
ROC_LMT_BASE_ID_GET(lmt_base, lmt_id);
inst = (struct cpt_inst_s *)lmt_base;
again:
for (i = 0; i < RTE_MIN(PKTS_PER_LOOP, nb_ops); i++) {
infl_req = &pend_q->req_queue[pend_q->enq_tail];
infl_req->op_flags = 0;
ret = cn10k_cpt_fill_inst(qp, ops + i, &inst[2 * i], infl_req);
if (unlikely(ret != 1)) {
plt_dp_err("Could not process op: %p", ops + i);
if (i == 0)
goto update_pending;
break;
}
MOD_INC(pend_q->enq_tail, qp->lf.nb_desc);
}
if (i > PKTS_PER_STEORL) {
lmt_arg = ROC_CN10K_CPT_LMT_ARG | (PKTS_PER_STEORL - 1) << 12 |
(uint64_t)lmt_id;
roc_lmt_submit_steorl(lmt_arg, io_addr);
lmt_arg = ROC_CN10K_CPT_LMT_ARG |
(i - PKTS_PER_STEORL - 1) << 12 |
(uint64_t)(lmt_id + PKTS_PER_STEORL);
roc_lmt_submit_steorl(lmt_arg, io_addr);
} else {
lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - 1) << 12 |
(uint64_t)lmt_id;
roc_lmt_submit_steorl(lmt_arg, io_addr);
}
rte_io_wmb();
if (nb_ops - i > 0 && i == PKTS_PER_LOOP) {
nb_ops -= i;
ops += i;
count += i;
goto again;
}
update_pending:
pend_q->pending_count += count + i;
pend_q->time_out = rte_get_timer_cycles() +
DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz();
return count + i;
}
void
cn10k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev)
{
dev->enqueue_burst = cn10k_cpt_enqueue_burst;
rte_mb();
}
static void
cn10k_cpt_dev_info_get(struct rte_cryptodev *dev,

2
drivers/crypto/cnxk/cn10k_cryptodev_ops.h
View File

@ -10,4 +10,6 @@
extern struct rte_cryptodev_ops cn10k_cpt_ops;
void cn10k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev);
#endif /* _CN10K_CRYPTODEV_OPS_H_ */

2
drivers/crypto/cnxk/cn9k_cryptodev.c
View File

@ -78,6 +78,8 @@ cn9k_cpt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
dev->dev_ops = &cn9k_cpt_ops;
dev->driver_id = cn9k_cryptodev_driver_id;
cn9k_cpt_set_enqdeq_fns(dev);
return 0;
dev_fini:

154
drivers/crypto/cnxk/cn9k_cryptodev_ops.c
View File

@ -7,7 +7,161 @@
#include "cn9k_cryptodev.h"
#include "cn9k_cryptodev_ops.h"
#include "cnxk_cryptodev.h"
#include "cnxk_cryptodev_ops.h"
#include "cnxk_se.h"
static __rte_always_inline int __rte_hot
cn9k_cpt_sym_inst_fill(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op,
struct cnxk_se_sess *sess,
struct cpt_inflight_req *infl_req,
struct cpt_inst_s *inst)
{
RTE_SET_USED(qp);
RTE_SET_USED(op);
RTE_SET_USED(sess);
RTE_SET_USED(infl_req);
RTE_SET_USED(inst);
return -ENOTSUP;
}
static inline struct cnxk_se_sess *
cn9k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op)
{
const int driver_id = cn9k_cryptodev_driver_id;
struct rte_crypto_sym_op *sym_op = op->sym;
struct rte_cryptodev_sym_session *sess;
struct cnxk_se_sess *priv;
int ret;
/* Create temporary session */
sess = rte_cryptodev_sym_session_create(qp->sess_mp);
if (sess == NULL)
return NULL;
ret = sym_session_configure(qp->lf.roc_cpt, driver_id, sym_op->xform,
sess, qp->sess_mp_priv);
if (ret)
goto sess_put;
priv = get_sym_session_private_data(sess, driver_id);
sym_op->session = sess;
return priv;
sess_put:
rte_mempool_put(qp->sess_mp, sess);
return NULL;
}
static uint16_t
cn9k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
{
struct cpt_inflight_req *infl_req;
struct rte_crypto_sym_op *sym_op;
uint16_t nb_allowed, count = 0;
struct cnxk_cpt_qp *qp = qptr;
struct pending_queue *pend_q;
struct cnxk_se_sess *sess;
struct rte_crypto_op *op;
struct cpt_inst_s inst;
uint64_t lmt_status;
uint64_t lmtline;
uint64_t io_addr;
int ret;
pend_q = &qp->pend_q;
lmtline = qp->lmtline.lmt_base;
io_addr = qp->lmtline.io_addr;
inst.w0.u64 = 0;
inst.w2.u64 = 0;
inst.w3.u64 = 0;
nb_allowed = qp->lf.nb_desc - pend_q->pending_count;
nb_ops = RTE_MIN(nb_ops, nb_allowed);
for (count = 0; count < nb_ops; count++) {
op = ops[count];
infl_req = &pend_q->req_queue[pend_q->enq_tail];
infl_req->op_flags = 0;
if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) {
if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
sym_op = op->sym;
sess = get_sym_session_private_data(
sym_op->session,
cn9k_cryptodev_driver_id);
ret = cn9k_cpt_sym_inst_fill(qp, op, sess,
infl_req, &inst);
} else {
sess = cn9k_cpt_sym_temp_sess_create(qp, op);
if (unlikely(sess == NULL)) {
plt_dp_err(
"Could not create temp session");
break;
}
ret = cn9k_cpt_sym_inst_fill(qp, op, sess,
infl_req, &inst);
if (unlikely(ret)) {
sym_session_clear(
cn9k_cryptodev_driver_id,
op->sym->session);
rte_mempool_put(qp->sess_mp,
op->sym->session);
}
}
} else {
plt_dp_err("Unsupported op type");
break;
}
if (unlikely(ret)) {
plt_dp_err("Could not process op: %p", op);
break;
}
infl_req->cop = op;
infl_req->res.cn9k.compcode = CPT_COMP_NOT_DONE;
inst.res_addr = (uint64_t)&infl_req->res;
inst.w7.u64 = sess->cpt_inst_w7;
do {
/* Copy CPT command to LMTLINE */
memcpy((void *)lmtline, &inst, sizeof(inst));
/*
* Make sure compiler does not reorder memcpy and ldeor.
* LMTST transactions are always flushed from the write
* buffer immediately, a DMB is not required to push out
* LMTSTs.
*/
rte_io_wmb();
lmt_status = roc_lmt_submit_ldeor(io_addr);
} while (lmt_status == 0);
MOD_INC(pend_q->enq_tail, qp->lf.nb_desc);
}
pend_q->pending_count += count;
pend_q->time_out = rte_get_timer_cycles() +
DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz();
return count;
}
void
cn9k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev)
{
dev->enqueue_burst = cn9k_cpt_enqueue_burst;
rte_mb();
}
static void
cn9k_cpt_dev_info_get(struct rte_cryptodev *dev,

2
drivers/crypto/cnxk/cn9k_cryptodev_ops.h
View File

@ -9,4 +9,6 @@
extern struct rte_cryptodev_ops cn9k_cpt_ops;
void cn9k_cpt_set_enqdeq_fns(struct rte_cryptodev *dev);
#endif /* _CN9K_CRYPTODEV_OPS_H_ */

9
drivers/crypto/cnxk/cnxk_cryptodev_ops.h
View File

@ -11,6 +11,11 @@
#define CNXK_CPT_MIN_HEADROOM_REQ 24
/* Default command timeout in seconds */
#define DEFAULT_COMMAND_TIMEOUT 4
#define MOD_INC(i, l) ((i) == (l - 1) ? (i) = 0 : (i)++)
struct cpt_qp_meta_info {
struct rte_mempool *pool;
int mlen;
@ -26,6 +31,10 @@ enum sym_xform_type {
CNXK_CPT_CIPHER_DEC_AUTH_VRFY
};
#define CPT_OP_FLAGS_METABUF (1 << 1)
#define CPT_OP_FLAGS_AUTH_VERIFY (1 << 0)
#define CPT_OP_FLAGS_IPSEC_DIR_INBOUND (1 << 2)
struct cpt_inflight_req {
union cpt_res_s res;
struct rte_crypto_op *cop;