d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

crypto/cnxk: support event vectorization

Browse Source 
Add support for vector aggregation of crypto operations for cn10k.
Crypto operations will be grouped by sub event type, flow id, scheduler
type and queue id fields from  rte_event_crypto_metadata::response_info.

Signed-off-by: Volodymyr Fialko <vfialko@marvell.com>
This commit is contained in:
Volodymyr Fialko 2022-10-01 02:42:13 +02:00 committed by Akhil Goyal
parent c1749bc5ee
commit 96798dcfc2
9 changed files with 427 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

422
drivers/crypto/cnxk/cn10k_cryptodev_ops.c
View File

@ -19,6 +19,25 @@
#include "roc_api.h"
#define PKTS_PER_LOOP 32
#define PKTS_PER_STEORL 16
/* Holds information required to send crypto operations in one burst */
struct ops_burst {
struct rte_crypto_op *op[PKTS_PER_LOOP];
uint64_t w2[PKTS_PER_LOOP];
struct cn10k_sso_hws *ws;
struct cnxk_cpt_qp *qp;
uint16_t nb_ops;
};
/* Holds information required to send vector of operations */
struct vec_request {
struct cpt_inflight_req *req;
struct rte_event_vector *vec;
uint64_t w2;
};
static inline struct cnxk_se_sess *
cn10k_cpt_sym_temp_sess_create(struct cnxk_cpt_qp *qp, struct rte_crypto_op *op)
{
@ -164,9 +183,6 @@ cn10k_cpt_fill_inst(struct cnxk_cpt_qp *qp, struct rte_crypto_op *ops[],
return 1;
}
#define PKTS_PER_LOOP 32
#define PKTS_PER_STEORL 16
static uint16_t
cn10k_cpt_enqueue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
{
@ -267,9 +283,9 @@ cn10k_cpt_crypto_adapter_ev_mdata_set(struct rte_cryptodev *dev __rte_unused,
union rte_event_crypto_metadata *ec_mdata = mdata;
struct rte_event *rsp_info;
struct cnxk_cpt_qp *qp;
uint64_t w2, tag_type;
uint8_t cdev_id;
int16_t qp_id;
uint64_t w2;
/* Get queue pair */
cdev_id = ec_mdata->request_info.cdev_id;
@ -277,9 +293,9 @@ cn10k_cpt_crypto_adapter_ev_mdata_set(struct rte_cryptodev *dev __rte_unused,
qp = rte_cryptodevs[cdev_id].data->queue_pairs[qp_id];
/* Prepare w2 */
tag_type = qp->ca.vector_sz ? RTE_EVENT_TYPE_CRYPTODEV_VECTOR : RTE_EVENT_TYPE_CRYPTODEV;
rsp_info = &ec_mdata->response_info;
w2 = CNXK_CPT_INST_W2((RTE_EVENT_TYPE_CRYPTODEV << 28) |
(rsp_info->sub_event_type << 20) |
w2 = CNXK_CPT_INST_W2((tag_type << 28) | (rsp_info->sub_event_type << 20) |
rsp_info->flow_id,
rsp_info->sched_type, rsp_info->queue_id, 0);
@ -373,18 +389,110 @@ cn10k_ca_meta_info_extract(struct rte_crypto_op *op,
return 0;
}
static inline uint16_t
ca_lmtst_burst_submit(struct cn10k_sso_hws *ws, uint64_t w2[], struct cnxk_cpt_qp *qp,
struct rte_crypto_op *op[], uint16_t nb_ops)
static inline void
cn10k_cpt_vec_inst_fill(struct vec_request *vec_req, struct cpt_inst_s *inst,
struct cnxk_cpt_qp *qp)
{
const union cpt_res_s res = {.cn10k.compcode = CPT_COMP_NOT_DONE};
struct cpt_inflight_req *infl_req = vec_req->req;
const union cpt_inst_w4 w4 = {
.s.opcode_major = ROC_SE_MAJOR_OP_MISC,
.s.opcode_minor = ROC_SE_MISC_MINOR_OP_PASSTHROUGH,
.s.param1 = 1,
.s.param2 = 1,
.s.dlen = 0,
};
infl_req->vec = vec_req->vec;
infl_req->qp = qp;
inst->res_addr = (uint64_t)&infl_req->res;
__atomic_store_n(&infl_req->res.u64[0], res.u64[0], __ATOMIC_RELAXED);
inst->w0.u64 = 0;
inst->w2.u64 = vec_req->w2;
inst->w3.u64 = CNXK_CPT_INST_W3(1, infl_req);
inst->w4.u64 = w4.u64;
inst->w7.u64 = ROC_CPT_DFLT_ENG_GRP_SE << 61;
}
static void
cn10k_cpt_vec_pkt_submission_timeout_handle(void)
{
plt_dp_err("Vector packet submission timedout");
abort();
}
static inline void
cn10k_cpt_vec_submit(struct vec_request vec_tbl[], uint16_t vec_tbl_len, struct cnxk_cpt_qp *qp)
{
uint64_t lmt_base, lmt_arg, lmt_id, io_addr;
union cpt_fc_write_s fc;
struct cpt_inst_s *inst;
uint16_t burst_size;
uint64_t *fc_addr;
int i;
if (vec_tbl_len == 0)
return;
const uint32_t fc_thresh = qp->lmtline.fc_thresh;
/*
* Use 10 mins timeout for the poll. It is not possible to recover from partial submission
* of vector packet. Actual packets for processing are submitted to CPT prior to this
* routine. Hence, any failure for submission of vector packet would indicate an
* unrecoverable error for the application.
*/
const uint64_t timeout = rte_get_timer_cycles() + 10 * 60 * rte_get_timer_hz();
lmt_base = qp->lmtline.lmt_base;
io_addr = qp->lmtline.io_addr;
fc_addr = qp->lmtline.fc_addr;
ROC_LMT_BASE_ID_GET(lmt_base, lmt_id);
inst = (struct cpt_inst_s *)lmt_base;
again:
burst_size = RTE_MIN(PKTS_PER_STEORL, vec_tbl_len);
for (i = 0; i < burst_size; i++)
cn10k_cpt_vec_inst_fill(&vec_tbl[i], &inst[i * 2], qp);
do {
fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED);
if (likely(fc.s.qsize < fc_thresh))
break;
if (unlikely(rte_get_timer_cycles() > timeout))
cn10k_cpt_vec_pkt_submission_timeout_handle();
} while (true);
lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - 1) << 12 | lmt_id;
roc_lmt_submit_steorl(lmt_arg, io_addr);
rte_io_wmb();
vec_tbl_len -= i;
if (vec_tbl_len > 0) {
vec_tbl += i;
goto again;
}
}
static inline int
ca_lmtst_vec_submit(struct ops_burst *burst, struct vec_request vec_tbl[], uint16_t *vec_tbl_len)
{
struct cpt_inflight_req *infl_reqs[PKTS_PER_LOOP];
uint64_t lmt_base, lmt_arg, io_addr;
uint16_t lmt_id, len = *vec_tbl_len;
struct cpt_inst_s *inst, *inst_base;
struct cpt_inflight_req *infl_req;
struct rte_event_vector *vec;
union cpt_fc_write_s fc;
struct cnxk_cpt_qp *qp;
uint64_t *fc_addr;
uint16_t lmt_id;
int ret, i;
int ret, i, vi;
qp = burst->qp;
lmt_base = qp->lmtline.lmt_base;
io_addr = qp->lmtline.io_addr;
@ -395,24 +503,33 @@ ca_lmtst_burst_submit(struct cn10k_sso_hws *ws, uint64_t w2[], struct cnxk_cpt_q
ROC_LMT_BASE_ID_GET(lmt_base, lmt_id);
inst_base = (struct cpt_inst_s *)lmt_base;
#ifdef CNXK_CRYPTODEV_DEBUG
if (unlikely(!qp->ca.enabled)) {
rte_errno = EINVAL;
return 0;
}
#endif
if (unlikely(rte_mempool_get_bulk(qp->ca.req_mp, (void **)infl_reqs, nb_ops))) {
/* Perform fc check before putting packets into vectors */
fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED);
if (unlikely(fc.s.qsize > fc_thresh)) {
rte_errno = EAGAIN;
return 0;
}
if (unlikely(rte_mempool_get_bulk(qp->ca.req_mp, (void **)infl_reqs, burst->nb_ops))) {
rte_errno = ENOMEM;
return 0;
}
for (i = 0; i < nb_ops; i++) {
for (i = 0; i < burst->nb_ops; i++) {
inst = &inst_base[2 * i];
infl_req = infl_reqs[i];
infl_req->op_flags = 0;
ret = cn10k_cpt_fill_inst(qp, &op[i], inst, infl_req);
ret = cn10k_cpt_fill_inst(qp, &burst->op[i], inst, infl_req);
if (unlikely(ret != 1)) {
plt_dp_dbg("Could not process op: %p", op[i]);
plt_cpt_dbg("Could not process op: %p", burst->op[i]);
if (i != 0)
goto submit;
else
@ -421,22 +538,50 @@ ca_lmtst_burst_submit(struct cn10k_sso_hws *ws, uint64_t w2[], struct cnxk_cpt_q
infl_req->res.cn10k.compcode = CPT_COMP_NOT_DONE;
infl_req->qp = qp;
inst->w0.u64 = 0;
inst->res_addr = (uint64_t)&infl_req->res;
inst->w2.u64 = w2[i];
inst->w3.u64 = CNXK_CPT_INST_W3(1, infl_req);
}
fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED);
if (unlikely(fc.s.qsize > fc_thresh)) {
rte_errno = EAGAIN;
i = 0;
goto put;
inst->w3.u64 = 0x1;
/* Lookup for existing vector by w2 */
for (vi = len - 1; vi >= 0; vi--) {
if (vec_tbl[vi].w2 != burst->w2[i])
continue;
vec = vec_tbl[vi].vec;
if (unlikely(vec->nb_elem == qp->ca.vector_sz))
continue;
vec->ptrs[vec->nb_elem++] = infl_req;
goto next_op; /* continue outer loop */
}
/* No available vectors found, allocate a new one */
if (unlikely(rte_mempool_get(qp->ca.vector_mp, (void **)&vec_tbl[len].vec))) {
rte_errno = ENOMEM;
if (i != 0)
goto submit;
else
goto put;
}
/* Also preallocate in-flight request, that will be used to
* submit misc passthrough instruction
*/
if (unlikely(rte_mempool_get(qp->ca.req_mp, (void **)&vec_tbl[len].req))) {
rte_mempool_put(qp->ca.vector_mp, vec_tbl[len].vec);
rte_errno = ENOMEM;
if (i != 0)
goto submit;
else
goto put;
}
vec_tbl[len].w2 = burst->w2[i];
vec_tbl[len].vec->ptrs[0] = infl_req;
vec_tbl[len].vec->nb_elem = 1;
len++;
next_op:;
}
/* Submit operations in burst */
submit:
if (CNXK_TT_FROM_TAG(ws->gw_rdata) == SSO_TT_ORDERED)
roc_sso_hws_head_wait(ws->base);
if (CNXK_TT_FROM_TAG(burst->ws->gw_rdata) == SSO_TT_ORDERED)
roc_sso_hws_head_wait(burst->ws->base);
if (i > PKTS_PER_STEORL) {
lmt_arg = ROC_CN10K_CPT_LMT_ARG | (PKTS_PER_STEORL - 1) << 12 | (uint64_t)lmt_id;
@ -452,8 +597,104 @@ ca_lmtst_burst_submit(struct cn10k_sso_hws *ws, uint64_t w2[], struct cnxk_cpt_q
rte_io_wmb();
put:
if (unlikely(i != nb_ops))
rte_mempool_put_bulk(qp->ca.req_mp, (void *)&infl_reqs[i], nb_ops - i);
if (i != burst->nb_ops)
rte_mempool_put_bulk(qp->ca.req_mp, (void *)&infl_reqs[i], burst->nb_ops - i);
*vec_tbl_len = len;
return i;
}
static inline uint16_t
ca_lmtst_burst_submit(struct ops_burst *burst)
{
struct cpt_inflight_req *infl_reqs[PKTS_PER_LOOP];
uint64_t lmt_base, lmt_arg, io_addr;
struct cpt_inst_s *inst, *inst_base;
struct cpt_inflight_req *infl_req;
union cpt_fc_write_s fc;
struct cnxk_cpt_qp *qp;
uint64_t *fc_addr;
uint16_t lmt_id;
int ret, i, j;
qp = burst->qp;
lmt_base = qp->lmtline.lmt_base;
io_addr = qp->lmtline.io_addr;
fc_addr = qp->lmtline.fc_addr;
const uint32_t fc_thresh = qp->lmtline.fc_thresh;
ROC_LMT_BASE_ID_GET(lmt_base, lmt_id);
inst_base = (struct cpt_inst_s *)lmt_base;
#ifdef CNXK_CRYPTODEV_DEBUG
if (unlikely(!qp->ca.enabled)) {
rte_errno = EINVAL;
return 0;
}
#endif
if (unlikely(rte_mempool_get_bulk(qp->ca.req_mp, (void **)infl_reqs, burst->nb_ops))) {
rte_errno = ENOMEM;
return 0;
}
for (i = 0; i < burst->nb_ops; i++) {
inst = &inst_base[2 * i];
infl_req = infl_reqs[i];
infl_req->op_flags = 0;
ret = cn10k_cpt_fill_inst(qp, &burst->op[i], inst, infl_req);
if (unlikely(ret != 1)) {
plt_dp_dbg("Could not process op: %p", burst->op[i]);
if (i != 0)
goto submit;
else
goto put;
}
infl_req->res.cn10k.compcode = CPT_COMP_NOT_DONE;
infl_req->qp = qp;
inst->w0.u64 = 0;
inst->res_addr = (uint64_t)&infl_req->res;
inst->w2.u64 = burst->w2[i];
inst->w3.u64 = CNXK_CPT_INST_W3(1, infl_req);
}
fc.u64[0] = __atomic_load_n(fc_addr, __ATOMIC_RELAXED);
if (unlikely(fc.s.qsize > fc_thresh)) {
rte_errno = EAGAIN;
for (j = 0; j < i; j++) {
infl_req = infl_reqs[j];
if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF))
rte_mempool_put(qp->meta_info.pool, infl_req->mdata);
}
i = 0;
goto put;
}
submit:
if (CNXK_TT_FROM_TAG(burst->ws->gw_rdata) == SSO_TT_ORDERED)
roc_sso_hws_head_wait(burst->ws->base);
if (i > PKTS_PER_STEORL) {
lmt_arg = ROC_CN10K_CPT_LMT_ARG | (PKTS_PER_STEORL - 1) << 12 | (uint64_t)lmt_id;
roc_lmt_submit_steorl(lmt_arg, io_addr);
lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - PKTS_PER_STEORL - 1) << 12 |
(uint64_t)(lmt_id + PKTS_PER_STEORL);
roc_lmt_submit_steorl(lmt_arg, io_addr);
} else {
lmt_arg = ROC_CN10K_CPT_LMT_ARG | (i - 1) << 12 | (uint64_t)lmt_id;
roc_lmt_submit_steorl(lmt_arg, io_addr);
}
rte_io_wmb();
put:
if (unlikely(i != burst->nb_ops))
rte_mempool_put_bulk(qp->ca.req_mp, (void *)&infl_reqs[i], burst->nb_ops - i);
return i;
}
@ -461,42 +702,76 @@ ca_lmtst_burst_submit(struct cn10k_sso_hws *ws, uint64_t w2[], struct cnxk_cpt_q
uint16_t __rte_hot
cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event ev[], uint16_t nb_events)
{
struct rte_crypto_op *ops[PKTS_PER_LOOP], *op;
struct cnxk_cpt_qp *qp, *curr_qp = NULL;
uint64_t w2s[PKTS_PER_LOOP], w2;
uint16_t submitted, count = 0;
int ret, i, ops_len = 0;
uint16_t submitted, count = 0, vec_tbl_len = 0;
struct vec_request vec_tbl[nb_events];
struct rte_crypto_op *op;
struct ops_burst burst;
struct cnxk_cpt_qp *qp;
bool is_vector = false;
uint64_t w2;
int ret, i;
burst.ws = ws;
burst.qp = NULL;
burst.nb_ops = 0;
for (i = 0; i < nb_events; i++) {
op = ev[i].event_ptr;
ret = cn10k_ca_meta_info_extract(op, &qp, &w2);
if (unlikely(ret)) {
rte_errno = EINVAL;
return count;
goto vec_submit;
}
if (qp != curr_qp) {
if (ops_len) {
submitted = ca_lmtst_burst_submit(ws, w2s, curr_qp, ops, ops_len);
/* Queue pair change check */
if (qp != burst.qp) {
if (burst.nb_ops) {
if (is_vector) {
submitted =
ca_lmtst_vec_submit(&burst, vec_tbl, &vec_tbl_len);
/*
* Vector submission is required on qp change, but not in
* other cases, since we could send several vectors per
* lmtst instruction only for same qp
*/
cn10k_cpt_vec_submit(vec_tbl, vec_tbl_len, burst.qp);
vec_tbl_len = 0;
} else {
submitted = ca_lmtst_burst_submit(&burst);
}
count += submitted;
if (unlikely(submitted != ops_len))
return count;
ops_len = 0;
if (unlikely(submitted != burst.nb_ops))
goto vec_submit;
burst.nb_ops = 0;
}
curr_qp = qp;
is_vector = qp->ca.vector_sz;
burst.qp = qp;
}
w2s[ops_len] = w2;
ops[ops_len] = op;
if (++ops_len == PKTS_PER_LOOP) {
submitted = ca_lmtst_burst_submit(ws, w2s, curr_qp, ops, ops_len);
burst.w2[burst.nb_ops] = w2;
burst.op[burst.nb_ops] = op;
/* Max nb_ops per burst check */
if (++burst.nb_ops == PKTS_PER_LOOP) {
if (is_vector)
submitted = ca_lmtst_vec_submit(&burst, vec_tbl, &vec_tbl_len);
else
submitted = ca_lmtst_burst_submit(&burst);
count += submitted;
if (unlikely(submitted != ops_len))
return count;
ops_len = 0;
if (unlikely(submitted != burst.nb_ops))
goto vec_submit;
burst.nb_ops = 0;
}
}
if (ops_len)
count += ca_lmtst_burst_submit(ws, w2s, curr_qp, ops, ops_len);
/* Submit the rest of crypto operations */
if (burst.nb_ops) {
if (is_vector)
count += ca_lmtst_vec_submit(&burst, vec_tbl, &vec_tbl_len);
else
count += ca_lmtst_burst_submit(&burst);
}
vec_submit:
cn10k_cpt_vec_submit(vec_tbl, vec_tbl_len, burst.qp);
return count;
}
@ -654,6 +929,49 @@ cn10k_cpt_crypto_adapter_dequeue(uintptr_t get_work1)
return (uintptr_t)cop;
}
uintptr_t
cn10k_cpt_crypto_adapter_vector_dequeue(uintptr_t get_work1)
{
struct cpt_inflight_req *infl_req, *vec_infl_req;
struct rte_mempool *meta_mp, *req_mp;
struct rte_event_vector *vec;
struct rte_crypto_op *cop;
struct cnxk_cpt_qp *qp;
union cpt_res_s res;
int i;
vec_infl_req = (struct cpt_inflight_req *)(get_work1);
vec = vec_infl_req->vec;
qp = vec_infl_req->qp;
meta_mp = qp->meta_info.pool;
req_mp = qp->ca.req_mp;
#ifdef CNXK_CRYPTODEV_DEBUG
res.u64[0] = __atomic_load_n(&vec_infl_req->res.u64[0], __ATOMIC_RELAXED);
PLT_ASSERT(res.cn10k.compcode == CPT_COMP_WARN);
PLT_ASSERT(res.cn10k.uc_compcode == 0);
#endif
for (i = 0; i < vec->nb_elem; i++) {
infl_req = vec->ptrs[i];
cop = infl_req->cop;
res.u64[0] = __atomic_load_n(&infl_req->res.u64[0], __ATOMIC_RELAXED);
cn10k_cpt_dequeue_post_process(qp, cop, infl_req, &res.cn10k);
vec->ptrs[i] = cop;
if (unlikely(infl_req->op_flags & CPT_OP_FLAGS_METABUF))
rte_mempool_put(meta_mp, infl_req->mdata);
rte_mempool_put(req_mp, infl_req);
}
rte_mempool_put(req_mp, vec_infl_req);
return (uintptr_t)vec;
}
static uint16_t
cn10k_cpt_dequeue_burst(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
{

2
drivers/crypto/cnxk/cn10k_cryptodev_ops.h
View File

@ -18,5 +18,7 @@ uint16_t __rte_hot cn10k_cpt_crypto_adapter_enqueue(void *ws, struct rte_event e
uint16_t nb_events);
__rte_internal
uintptr_t cn10k_cpt_crypto_adapter_dequeue(uintptr_t get_work1);
__rte_internal
uintptr_t cn10k_cpt_crypto_adapter_vector_dequeue(uintptr_t get_work1);
#endif /* _CN10K_CRYPTODEV_OPS_H_ */

9
drivers/crypto/cnxk/cnxk_cryptodev_ops.h
View File

@ -37,7 +37,10 @@ struct cpt_qp_meta_info {
struct cpt_inflight_req {
union cpt_res_s res;
struct rte_crypto_op *cop;
union {
struct rte_crypto_op *cop;
struct rte_event_vector *vec;
};
void *mdata;
uint8_t op_flags;
void *qp;
@ -63,6 +66,10 @@ struct crypto_adpter_info {
/**< Set if queue pair is added to crypto adapter */
struct rte_mempool *req_mp;
/**< CPT inflight request mempool */
uint16_t vector_sz;
/** Maximum number of cops to combine into single vector */
struct rte_mempool *vector_mp;
/** Pool for allocating rte_event_vector */
};
struct cnxk_cpt_qp {

1
drivers/crypto/cnxk/version.map
View File

@ -5,6 +5,7 @@ INTERNAL {
cn9k_cpt_crypto_adapter_dequeue;
cn10k_cpt_crypto_adapter_enqueue;
cn10k_cpt_crypto_adapter_dequeue;
cn10k_cpt_crypto_adapter_vector_dequeue;
local: *;
};

31
drivers/event/cnxk/cn10k_eventdev.c
View File

@ -1025,7 +1025,8 @@ cn10k_crypto_adapter_caps_get(const struct rte_eventdev *event_dev,
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k");
*caps = RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD |
RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA;
RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA |
RTE_EVENT_CRYPTO_ADAPTER_CAP_EVENT_VECTOR;
return 0;
}
@ -1039,23 +1040,20 @@ cn10k_crypto_adapter_qp_add(const struct rte_eventdev *event_dev,
struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
int ret;
RTE_SET_USED(conf);
CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k");
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k");
dev->is_ca_internal_port = 1;
cn10k_sso_fp_fns_set((struct rte_eventdev *)(uintptr_t)event_dev);
ret = cnxk_crypto_adapter_qp_add(event_dev, cdev, queue_pair_id);
ret = cnxk_crypto_adapter_qp_add(event_dev, cdev, queue_pair_id, conf);
cn10k_sso_set_priv_mem(event_dev, NULL, 0);
return ret;
}
static int
cn10k_crypto_adapter_qp_del(const struct rte_eventdev *event_dev,
const struct rte_cryptodev *cdev,
cn10k_crypto_adapter_qp_del(const struct rte_eventdev *event_dev, const struct rte_cryptodev *cdev,
int32_t queue_pair_id)
{
CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k");
@ -1072,6 +1070,26 @@ cn10k_tim_caps_get(const struct rte_eventdev *evdev, uint64_t flags,
cn10k_sso_set_priv_mem);
}
static int
cn10k_crypto_adapter_vec_limits(const struct rte_eventdev *event_dev,
const struct rte_cryptodev *cdev,
struct rte_event_crypto_adapter_vector_limits *limits)
{
CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn10k");
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn10k");
limits->log2_sz = false;
limits->min_sz = 0;
limits->max_sz = UINT16_MAX;
/* Unused timeout, in software implementation we aggregate all crypto
* operations passed to the enqueue function
*/
limits->min_timeout_ns = 0;
limits->max_timeout_ns = 0;
return 0;
}
static struct eventdev_ops cn10k_sso_dev_ops = {
.dev_infos_get = cn10k_sso_info_get,
.dev_configure = cn10k_sso_dev_configure,
@ -1109,6 +1127,7 @@ static struct eventdev_ops cn10k_sso_dev_ops = {
.crypto_adapter_caps_get = cn10k_crypto_adapter_caps_get,
.crypto_adapter_queue_pair_add = cn10k_crypto_adapter_qp_add,
.crypto_adapter_queue_pair_del = cn10k_crypto_adapter_qp_del,
.crypto_adapter_vector_limits_get = cn10k_crypto_adapter_vec_limits,
.xstats_get = cnxk_sso_xstats_get,
.xstats_reset = cnxk_sso_xstats_reset,

6
drivers/event/cnxk/cn10k_worker.h
View File

@ -230,6 +230,9 @@ cn10k_sso_hws_post_process(struct cn10k_sso_hws *ws, uint64_t *u64,
if ((flags & CPT_RX_WQE_F) &&
(CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_CRYPTODEV)) {
u64[1] = cn10k_cpt_crypto_adapter_dequeue(u64[1]);
} else if ((flags & CPT_RX_WQE_F) &&
(CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_CRYPTODEV_VECTOR)) {
u64[1] = cn10k_cpt_crypto_adapter_vector_dequeue(u64[1]);
} else if (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_ETHDEV) {
uint8_t port = CNXK_SUB_EVENT_FROM_TAG(u64[0]);
uint64_t mbuf;
@ -272,8 +275,7 @@ cn10k_sso_hws_post_process(struct cn10k_sso_hws *ws, uint64_t *u64,
cn10k_sso_process_tstamp(u64[1], mbuf,
ws->tstamp[port]);
u64[1] = mbuf;
} else if (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) ==
RTE_EVENT_TYPE_ETHDEV_VECTOR) {
} else if (CNXK_EVENT_TYPE_FROM_TAG(u64[0]) == RTE_EVENT_TYPE_ETHDEV_VECTOR) {
uint8_t port = CNXK_SUB_EVENT_FROM_TAG(u64[0]);
__uint128_t vwqe_hdr = *(__uint128_t *)u64[1];

7
drivers/event/cnxk/cn9k_eventdev.c
View File

@ -1131,23 +1131,20 @@ cn9k_crypto_adapter_qp_add(const struct rte_eventdev *event_dev,
struct cnxk_sso_evdev *dev = cnxk_sso_pmd_priv(event_dev);
int ret;
RTE_SET_USED(conf);
CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn9k");
CNXK_VALID_DEV_OR_ERR_RET(cdev->device, "crypto_cn9k");
dev->is_ca_internal_port = 1;
cn9k_sso_fp_fns_set((struct rte_eventdev *)(uintptr_t)event_dev);
ret = cnxk_crypto_adapter_qp_add(event_dev, cdev, queue_pair_id);
ret = cnxk_crypto_adapter_qp_add(event_dev, cdev, queue_pair_id, conf);
cn9k_sso_set_priv_mem(event_dev, NULL, 0);
return ret;
}
static int
cn9k_crypto_adapter_qp_del(const struct rte_eventdev *event_dev,
const struct rte_cryptodev *cdev,
cn9k_crypto_adapter_qp_del(const struct rte_eventdev *event_dev, const struct rte_cryptodev *cdev,
int32_t queue_pair_id)
{
CNXK_VALID_DEV_OR_ERR_RET(event_dev->dev, "event_cn9k");

4
drivers/event/cnxk/cnxk_eventdev.h
View File

@ -10,6 +10,7 @@
#include <cryptodev_pmd.h>
#include <rte_devargs.h>
#include <rte_ethdev.h>
#include <rte_event_crypto_adapter.h>
#include <rte_event_eth_rx_adapter.h>
#include <rte_event_eth_tx_adapter.h>
#include <rte_kvargs.h>
@ -305,7 +306,8 @@ int cnxk_sso_tx_adapter_start(uint8_t id, const struct rte_eventdev *event_dev);
int cnxk_sso_tx_adapter_stop(uint8_t id, const struct rte_eventdev *event_dev);
int cnxk_sso_tx_adapter_free(uint8_t id, const struct rte_eventdev *event_dev);
int cnxk_crypto_adapter_qp_add(const struct rte_eventdev *event_dev,
const struct rte_cryptodev *cdev, int32_t queue_pair_id);
const struct rte_cryptodev *cdev, int32_t queue_pair_id,
const struct rte_event_crypto_adapter_queue_conf *conf);
int cnxk_crypto_adapter_qp_del(const struct rte_cryptodev *cdev, int32_t queue_pair_id);
#endif /* __CNXK_EVENTDEV_H__ */

17
drivers/event/cnxk/cnxk_eventdev_adptr.c
View File

@ -641,7 +641,8 @@ cnxk_sso_tx_adapter_free(uint8_t id __rte_unused,
}
static int
crypto_adapter_qp_setup(const struct rte_cryptodev *cdev, struct cnxk_cpt_qp *qp)
crypto_adapter_qp_setup(const struct rte_cryptodev *cdev, struct cnxk_cpt_qp *qp,
const struct rte_event_crypto_adapter_queue_conf *conf)
{
char name[RTE_MEMPOOL_NAMESIZE];
uint32_t cache_size, nb_req;
@ -674,6 +675,10 @@ crypto_adapter_qp_setup(const struct rte_cryptodev *cdev, struct cnxk_cpt_qp *qp
if (qp->ca.req_mp == NULL)
return -ENOMEM;
if (conf != NULL) {
qp->ca.vector_sz = conf->vector_sz;
qp->ca.vector_mp = conf->vector_mp;
}
qp->ca.enabled = true;
return 0;
@ -681,7 +686,8 @@ crypto_adapter_qp_setup(const struct rte_cryptodev *cdev, struct cnxk_cpt_qp *qp
int
cnxk_crypto_adapter_qp_add(const struct rte_eventdev *event_dev, const struct rte_cryptodev *cdev,
int32_t queue_pair_id)
int32_t queue_pair_id,
const struct rte_event_crypto_adapter_queue_conf *conf)
{
struct cnxk_sso_evdev *sso_evdev = cnxk_sso_pmd_priv(event_dev);
uint32_t adptr_xae_cnt = 0;
@ -693,7 +699,7 @@ cnxk_crypto_adapter_qp_add(const struct rte_eventdev *event_dev, const struct rt
for (qp_id = 0; qp_id < cdev->data->nb_queue_pairs; qp_id++) {
qp = cdev->data->queue_pairs[qp_id];
ret = crypto_adapter_qp_setup(cdev, qp);
ret = crypto_adapter_qp_setup(cdev, qp, conf);
if (ret) {
cnxk_crypto_adapter_qp_del(cdev, -1);
return ret;
@ -702,7 +708,7 @@ cnxk_crypto_adapter_qp_add(const struct rte_eventdev *event_dev, const struct rt
}
} else {
qp = cdev->data->queue_pairs[queue_pair_id];
ret = crypto_adapter_qp_setup(cdev, qp);
ret = crypto_adapter_qp_setup(cdev, qp, conf);
if (ret)
return ret;
adptr_xae_cnt = qp->ca.req_mp->size;
@ -733,7 +739,8 @@ crypto_adapter_qp_free(struct cnxk_cpt_qp *qp)
}
int
cnxk_crypto_adapter_qp_del(const struct rte_cryptodev *cdev, int32_t queue_pair_id)
cnxk_crypto_adapter_qp_del(const struct rte_cryptodev *cdev,
int32_t queue_pair_id)
{
struct cnxk_cpt_qp *qp;