crypto/qat: unify raw data path functions
This patch unifies QAT's raw dp api implementations to the same enqueue/dequeue methods used in crypto operations. The specific functions for different QAT generation are updated respectively. The qat_sym_hw_dp.c is removed as no longer required. Signed-off-by: Kai Ji <kai.ji@intel.com> Acked-by: Fan Zhang <roy.fan.zhang@intel.com>
This commit is contained in:
Kai Ji
Akhil Goyal
parent
fb3b9f4922
commit
85fec6fd96
@ -74,7 +74,7 @@ endif
|
|||||||
|
|
||||||
if qat_crypto
|
if qat_crypto
|
||||||
foreach f: ['qat_sym.c', 'qat_sym_session.c',
|
foreach f: ['qat_sym.c', 'qat_sym_session.c',
|
||||||
'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
|
'qat_asym.c', 'qat_crypto.c',
|
||||||
'dev/qat_sym_pmd_gen1.c',
|
'dev/qat_sym_pmd_gen1.c',
|
||||||
'dev/qat_asym_pmd_gen1.c',
|
'dev/qat_asym_pmd_gen1.c',
|
||||||
'dev/qat_crypto_pmd_gen2.c',
|
'dev/qat_crypto_pmd_gen2.c',
|
||||||
|
|||||||
@ -616,11 +616,18 @@ static struct rte_compressdev_ops compress_qat_dummy_ops = {
|
|||||||
.private_xform_free = qat_comp_private_xform_free
|
.private_xform_free = qat_comp_private_xform_free
|
||||||
};
|
};
|
||||||
|
|
||||||
|
static uint16_t
|
||||||
|
qat_comp_dequeue_burst(void *qp, struct rte_comp_op **ops, uint16_t nb_ops)
|
||||||
|
{
|
||||||
|
return qat_dequeue_op_burst(qp, (void **)ops, qat_comp_process_response,
|
||||||
|
nb_ops);
|
||||||
|
}
|
||||||
|
|
||||||
static uint16_t
|
static uint16_t
|
||||||
qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
|
qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
|
||||||
uint16_t nb_ops)
|
uint16_t nb_ops)
|
||||||
{
|
{
|
||||||
uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
|
uint16_t ret = qat_comp_dequeue_burst(qp, ops, nb_ops);
|
||||||
struct qat_qp *tmp_qp = (struct qat_qp *)qp;
|
struct qat_qp *tmp_qp = (struct qat_qp *)qp;
|
||||||
|
|
||||||
if (ret) {
|
if (ret) {
|
||||||
@ -638,8 +645,7 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
|
|||||||
|
|
||||||
} else {
|
} else {
|
||||||
tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
|
tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
|
||||||
(compressdev_dequeue_pkt_burst_t)
|
qat_comp_dequeue_burst;
|
||||||
qat_comp_pmd_enq_deq_dummy_op_burst;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return ret;
|
return ret;
|
||||||
|
|||||||
@ -291,6 +291,8 @@ RTE_INIT(qat_sym_crypto_gen2_init)
|
|||||||
qat_sym_crypto_cap_get_gen2;
|
qat_sym_crypto_cap_get_gen2;
|
||||||
qat_sym_gen_dev_ops[QAT_GEN2].set_session =
|
qat_sym_gen_dev_ops[QAT_GEN2].set_session =
|
||||||
qat_sym_crypto_set_session_gen2;
|
qat_sym_crypto_set_session_gen2;
|
||||||
|
qat_sym_gen_dev_ops[QAT_GEN2].set_raw_dp_ctx =
|
||||||
|
qat_sym_configure_raw_dp_ctx_gen1;
|
||||||
qat_sym_gen_dev_ops[QAT_GEN2].get_feature_flags =
|
qat_sym_gen_dev_ops[QAT_GEN2].get_feature_flags =
|
||||||
qat_sym_crypto_feature_flags_get_gen1;
|
qat_sym_crypto_feature_flags_get_gen1;
|
||||||
|
|
||||||
|
|||||||
@ -394,6 +394,218 @@ qat_sym_crypto_set_session_gen3(void *cdev __rte_unused, void *session)
|
|||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
qat_sym_dp_enqueue_single_aead_gen3(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
||||||
|
union rte_crypto_sym_ofs ofs,
|
||||||
|
struct rte_crypto_va_iova_ptr *iv,
|
||||||
|
struct rte_crypto_va_iova_ptr *digest,
|
||||||
|
struct rte_crypto_va_iova_ptr *aad,
|
||||||
|
void *user_data)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_op_cookie *cookie;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
|
||||||
|
int32_t data_len;
|
||||||
|
uint32_t tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
cookie = qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
|
||||||
|
data, n_data_vecs, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
enqueue_one_aead_job_gen3(ctx, req, iv, digest, aad, ofs,
|
||||||
|
(uint32_t)data_len);
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue++;
|
||||||
|
|
||||||
|
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
|
||||||
|
qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv,
|
||||||
|
NULL, aad, digest);
|
||||||
|
#endif
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint32_t
|
||||||
|
qat_sym_dp_enqueue_aead_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
||||||
|
void *user_data[], int *status)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
uint32_t i, n;
|
||||||
|
uint32_t tail;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
int32_t data_len;
|
||||||
|
|
||||||
|
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
|
||||||
|
if (unlikely(n == 0)) {
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
|
||||||
|
*status = 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
for (i = 0; i < n; i++) {
|
||||||
|
struct qat_sym_op_cookie *cookie =
|
||||||
|
qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
|
||||||
|
vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
break;
|
||||||
|
|
||||||
|
enqueue_one_aead_job_gen3(ctx, req, &vec->iv[i],
|
||||||
|
&vec->digest[i], &vec->aad[i], ofs,
|
||||||
|
(uint32_t)data_len);
|
||||||
|
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
|
||||||
|
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
|
||||||
|
qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
|
||||||
|
vec->src_sgl[i].num, &vec->iv[i], NULL,
|
||||||
|
&vec->aad[i], &vec->digest[i]);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
if (unlikely(i < n))
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue += i;
|
||||||
|
*status = 0;
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
qat_sym_dp_enqueue_single_auth_gen3(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
||||||
|
union rte_crypto_sym_ofs ofs,
|
||||||
|
struct rte_crypto_va_iova_ptr *iv __rte_unused,
|
||||||
|
struct rte_crypto_va_iova_ptr *digest,
|
||||||
|
struct rte_crypto_va_iova_ptr *auth_iv,
|
||||||
|
void *user_data)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_op_cookie *cookie;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
int32_t data_len;
|
||||||
|
uint32_t tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
cookie = qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
|
||||||
|
data, n_data_vecs, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
enqueue_one_auth_job_gen3(ctx, cookie, req, digest, auth_iv, ofs,
|
||||||
|
(uint32_t)data_len);
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue++;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint32_t
|
||||||
|
qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
||||||
|
void *user_data[], int *status)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
uint32_t i, n;
|
||||||
|
uint32_t tail;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
int32_t data_len;
|
||||||
|
|
||||||
|
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
|
||||||
|
if (unlikely(n == 0)) {
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
|
||||||
|
*status = 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
for (i = 0; i < n; i++) {
|
||||||
|
struct qat_sym_op_cookie *cookie =
|
||||||
|
qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
|
||||||
|
vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
break;
|
||||||
|
enqueue_one_auth_job_gen3(ctx, cookie, req, &vec->digest[i],
|
||||||
|
&vec->auth_iv[i], ofs, (uint32_t)data_len);
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (unlikely(i < n))
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue += i;
|
||||||
|
*status = 0;
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
qat_sym_configure_raw_dp_ctx_gen3(void *_raw_dp_ctx, void *_ctx)
|
||||||
|
{
|
||||||
|
struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;
|
||||||
|
struct qat_sym_session *ctx = _ctx;
|
||||||
|
int ret;
|
||||||
|
|
||||||
|
ret = qat_sym_configure_raw_dp_ctx_gen1(_raw_dp_ctx, _ctx);
|
||||||
|
if (ret < 0)
|
||||||
|
return ret;
|
||||||
|
|
||||||
|
if (ctx->is_single_pass) {
|
||||||
|
raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_aead_jobs_gen3;
|
||||||
|
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead_gen3;
|
||||||
|
} else if (ctx->is_single_pass_gmac) {
|
||||||
|
raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs_gen3;
|
||||||
|
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth_gen3;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
RTE_INIT(qat_sym_crypto_gen3_init)
|
RTE_INIT(qat_sym_crypto_gen3_init)
|
||||||
{
|
{
|
||||||
qat_sym_gen_dev_ops[QAT_GEN3].cryptodev_ops = &qat_sym_crypto_ops_gen1;
|
qat_sym_gen_dev_ops[QAT_GEN3].cryptodev_ops = &qat_sym_crypto_ops_gen1;
|
||||||
@ -403,6 +615,8 @@ RTE_INIT(qat_sym_crypto_gen3_init)
|
|||||||
qat_sym_crypto_feature_flags_get_gen1;
|
qat_sym_crypto_feature_flags_get_gen1;
|
||||||
qat_sym_gen_dev_ops[QAT_GEN3].set_session =
|
qat_sym_gen_dev_ops[QAT_GEN3].set_session =
|
||||||
qat_sym_crypto_set_session_gen3;
|
qat_sym_crypto_set_session_gen3;
|
||||||
|
qat_sym_gen_dev_ops[QAT_GEN3].set_raw_dp_ctx =
|
||||||
|
qat_sym_configure_raw_dp_ctx_gen3;
|
||||||
#ifdef RTE_LIB_SECURITY
|
#ifdef RTE_LIB_SECURITY
|
||||||
qat_sym_gen_dev_ops[QAT_GEN3].create_security_ctx =
|
qat_sym_gen_dev_ops[QAT_GEN3].create_security_ctx =
|
||||||
qat_sym_create_security_gen1;
|
qat_sym_create_security_gen1;
|
||||||
|
|||||||
@ -223,6 +223,126 @@ qat_sym_crypto_set_session_gen4(void *cdev, void *session)
|
|||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
qat_sym_dp_enqueue_single_aead_gen4(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
||||||
|
union rte_crypto_sym_ofs ofs,
|
||||||
|
struct rte_crypto_va_iova_ptr *iv,
|
||||||
|
struct rte_crypto_va_iova_ptr *digest,
|
||||||
|
struct rte_crypto_va_iova_ptr *aad,
|
||||||
|
void *user_data)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_op_cookie *cookie;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
|
||||||
|
int32_t data_len;
|
||||||
|
uint32_t tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
cookie = qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
|
||||||
|
data, n_data_vecs, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
enqueue_one_aead_job_gen4(ctx, req, iv, digest, aad, ofs,
|
||||||
|
(uint32_t)data_len);
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue++;
|
||||||
|
|
||||||
|
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
|
||||||
|
qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv,
|
||||||
|
NULL, aad, digest);
|
||||||
|
#endif
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static uint32_t
|
||||||
|
qat_sym_dp_enqueue_aead_jobs_gen4(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
||||||
|
void *user_data[], int *status)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
uint32_t i, n;
|
||||||
|
uint32_t tail;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
int32_t data_len;
|
||||||
|
|
||||||
|
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
|
||||||
|
if (unlikely(n == 0)) {
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
|
||||||
|
*status = 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
for (i = 0; i < n; i++) {
|
||||||
|
struct qat_sym_op_cookie *cookie =
|
||||||
|
qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
|
||||||
|
vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
break;
|
||||||
|
|
||||||
|
enqueue_one_aead_job_gen4(ctx, req, &vec->iv[i],
|
||||||
|
&vec->digest[i], &vec->aad[i], ofs,
|
||||||
|
(uint32_t)data_len);
|
||||||
|
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
|
||||||
|
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
|
||||||
|
qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
|
||||||
|
vec->src_sgl[i].num, &vec->iv[i], NULL,
|
||||||
|
&vec->aad[i], &vec->digest[i]);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
if (unlikely(i < n))
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue += i;
|
||||||
|
*status = 0;
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
qat_sym_configure_raw_dp_ctx_gen4(void *_raw_dp_ctx, void *_ctx)
|
||||||
|
{
|
||||||
|
struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;
|
||||||
|
struct qat_sym_session *ctx = _ctx;
|
||||||
|
int ret;
|
||||||
|
|
||||||
|
ret = qat_sym_configure_raw_dp_ctx_gen1(_raw_dp_ctx, _ctx);
|
||||||
|
if (ret < 0)
|
||||||
|
return ret;
|
||||||
|
|
||||||
|
if (ctx->is_single_pass && ctx->is_ucs) {
|
||||||
|
raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_aead_jobs_gen4;
|
||||||
|
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead_gen4;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
RTE_INIT(qat_sym_crypto_gen4_init)
|
RTE_INIT(qat_sym_crypto_gen4_init)
|
||||||
{
|
{
|
||||||
qat_sym_gen_dev_ops[QAT_GEN4].cryptodev_ops = &qat_sym_crypto_ops_gen1;
|
qat_sym_gen_dev_ops[QAT_GEN4].cryptodev_ops = &qat_sym_crypto_ops_gen1;
|
||||||
@ -230,6 +350,8 @@ RTE_INIT(qat_sym_crypto_gen4_init)
|
|||||||
qat_sym_crypto_cap_get_gen4;
|
qat_sym_crypto_cap_get_gen4;
|
||||||
qat_sym_gen_dev_ops[QAT_GEN4].set_session =
|
qat_sym_gen_dev_ops[QAT_GEN4].set_session =
|
||||||
qat_sym_crypto_set_session_gen4;
|
qat_sym_crypto_set_session_gen4;
|
||||||
|
qat_sym_gen_dev_ops[QAT_GEN4].set_raw_dp_ctx =
|
||||||
|
qat_sym_configure_raw_dp_ctx_gen4;
|
||||||
qat_sym_gen_dev_ops[QAT_GEN4].get_feature_flags =
|
qat_sym_gen_dev_ops[QAT_GEN4].get_feature_flags =
|
||||||
qat_sym_crypto_feature_flags_get_gen1;
|
qat_sym_crypto_feature_flags_get_gen1;
|
||||||
#ifdef RTE_LIB_SECURITY
|
#ifdef RTE_LIB_SECURITY
|
||||||
|
|||||||
@ -839,6 +839,84 @@ int
|
|||||||
qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
|
qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
|
||||||
uint8_t *out_msg, void *op_cookie);
|
uint8_t *out_msg, void *op_cookie);
|
||||||
|
|
||||||
|
/* -----------------GEN 1 sym crypto raw data path APIs ---------------- */
|
||||||
|
int
|
||||||
|
qat_sym_dp_enqueue_single_cipher_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
||||||
|
union rte_crypto_sym_ofs ofs,
|
||||||
|
struct rte_crypto_va_iova_ptr *iv,
|
||||||
|
struct rte_crypto_va_iova_ptr *digest __rte_unused,
|
||||||
|
struct rte_crypto_va_iova_ptr *aad __rte_unused,
|
||||||
|
void *user_data);
|
||||||
|
|
||||||
|
uint32_t
|
||||||
|
qat_sym_dp_enqueue_cipher_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
||||||
|
void *user_data[], int *status);
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_dp_enqueue_single_auth_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
||||||
|
union rte_crypto_sym_ofs ofs,
|
||||||
|
struct rte_crypto_va_iova_ptr *iv __rte_unused,
|
||||||
|
struct rte_crypto_va_iova_ptr *digest,
|
||||||
|
struct rte_crypto_va_iova_ptr *auth_iv,
|
||||||
|
void *user_data);
|
||||||
|
|
||||||
|
uint32_t
|
||||||
|
qat_sym_dp_enqueue_auth_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
||||||
|
void *user_data[], int *status);
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_dp_enqueue_single_chain_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
||||||
|
union rte_crypto_sym_ofs ofs,
|
||||||
|
struct rte_crypto_va_iova_ptr *cipher_iv,
|
||||||
|
struct rte_crypto_va_iova_ptr *digest,
|
||||||
|
struct rte_crypto_va_iova_ptr *auth_iv,
|
||||||
|
void *user_data);
|
||||||
|
|
||||||
|
uint32_t
|
||||||
|
qat_sym_dp_enqueue_chain_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
||||||
|
void *user_data[], int *status);
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_dp_enqueue_single_aead_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
||||||
|
union rte_crypto_sym_ofs ofs,
|
||||||
|
struct rte_crypto_va_iova_ptr *iv,
|
||||||
|
struct rte_crypto_va_iova_ptr *digest,
|
||||||
|
struct rte_crypto_va_iova_ptr *aad,
|
||||||
|
void *user_data);
|
||||||
|
|
||||||
|
uint32_t
|
||||||
|
qat_sym_dp_enqueue_aead_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
||||||
|
void *user_data[], int *status);
|
||||||
|
|
||||||
|
void *
|
||||||
|
qat_sym_dp_dequeue_single_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
int *dequeue_status, enum rte_crypto_op_status *op_status);
|
||||||
|
|
||||||
|
uint32_t
|
||||||
|
qat_sym_dp_dequeue_burst_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
|
||||||
|
uint32_t max_nb_to_dequeue,
|
||||||
|
rte_cryptodev_raw_post_dequeue_t post_dequeue,
|
||||||
|
void **out_user_data, uint8_t is_user_data_array,
|
||||||
|
uint32_t *n_success_jobs, int *return_status);
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_dp_dequeue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx);
|
||||||
|
|
||||||
/* -----------------GENx control path APIs ---------------- */
|
/* -----------------GENx control path APIs ---------------- */
|
||||||
uint64_t
|
uint64_t
|
||||||
qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
|
qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
|
||||||
|
|||||||
@ -146,6 +146,10 @@ struct rte_cryptodev_ops qat_sym_crypto_ops_gen1 = {
|
|||||||
.sym_session_get_size = qat_sym_session_get_private_size,
|
.sym_session_get_size = qat_sym_session_get_private_size,
|
||||||
.sym_session_configure = qat_sym_session_configure,
|
.sym_session_configure = qat_sym_session_configure,
|
||||||
.sym_session_clear = qat_sym_session_clear,
|
.sym_session_clear = qat_sym_session_clear,
|
||||||
|
|
||||||
|
/* Raw data-path API related operations */
|
||||||
|
.sym_get_raw_dp_ctx_size = qat_sym_get_dp_ctx_size,
|
||||||
|
.sym_configure_raw_dp_ctx = qat_sym_configure_dp_ctx,
|
||||||
};
|
};
|
||||||
|
|
||||||
static struct qat_capabilities_info
|
static struct qat_capabilities_info
|
||||||
@ -448,6 +452,656 @@ qat_sym_create_security_gen1(void *cryptodev)
|
|||||||
}
|
}
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
int
|
||||||
|
qat_sym_dp_enqueue_single_cipher_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
||||||
|
union rte_crypto_sym_ofs ofs,
|
||||||
|
struct rte_crypto_va_iova_ptr *iv,
|
||||||
|
struct rte_crypto_va_iova_ptr *digest __rte_unused,
|
||||||
|
struct rte_crypto_va_iova_ptr *aad __rte_unused,
|
||||||
|
void *user_data)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
struct qat_sym_op_cookie *cookie;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
int32_t data_len;
|
||||||
|
uint32_t tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
cookie = qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
|
||||||
|
data, n_data_vecs, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
enqueue_one_cipher_job_gen1(ctx, req, iv, ofs, (uint32_t)data_len);
|
||||||
|
|
||||||
|
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
|
||||||
|
qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv,
|
||||||
|
NULL, NULL, NULL);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue++;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t
|
||||||
|
qat_sym_dp_enqueue_cipher_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
||||||
|
void *user_data[], int *status)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
uint32_t i, n;
|
||||||
|
uint32_t tail;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
int32_t data_len;
|
||||||
|
|
||||||
|
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
|
||||||
|
if (unlikely(n == 0)) {
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
|
||||||
|
*status = 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
for (i = 0; i < n; i++) {
|
||||||
|
struct qat_sym_op_cookie *cookie =
|
||||||
|
qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data[i],
|
||||||
|
cookie, vec->src_sgl[i].vec,
|
||||||
|
vec->src_sgl[i].num, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
break;
|
||||||
|
enqueue_one_cipher_job_gen1(ctx, req, &vec->iv[i], ofs,
|
||||||
|
(uint32_t)data_len);
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
|
||||||
|
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
|
||||||
|
qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
|
||||||
|
vec->src_sgl[i].num, &vec->iv[i],
|
||||||
|
NULL, NULL, NULL);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
if (unlikely(i < n))
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue += i;
|
||||||
|
*status = 0;
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_dp_enqueue_single_auth_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
||||||
|
union rte_crypto_sym_ofs ofs,
|
||||||
|
struct rte_crypto_va_iova_ptr *iv __rte_unused,
|
||||||
|
struct rte_crypto_va_iova_ptr *digest,
|
||||||
|
struct rte_crypto_va_iova_ptr *auth_iv,
|
||||||
|
void *user_data)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_op_cookie *cookie;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
int32_t data_len;
|
||||||
|
uint32_t tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
cookie = qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
|
||||||
|
data, n_data_vecs, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
enqueue_one_auth_job_gen1(ctx, req, digest, auth_iv, ofs,
|
||||||
|
(uint32_t)data_len);
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue++;
|
||||||
|
|
||||||
|
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
|
||||||
|
qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, NULL,
|
||||||
|
auth_iv, NULL, digest);
|
||||||
|
#endif
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t
|
||||||
|
qat_sym_dp_enqueue_auth_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
||||||
|
void *user_data[], int *status)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
uint32_t i, n;
|
||||||
|
uint32_t tail;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
int32_t data_len;
|
||||||
|
|
||||||
|
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
|
||||||
|
if (unlikely(n == 0)) {
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
|
||||||
|
*status = 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
for (i = 0; i < n; i++) {
|
||||||
|
struct qat_sym_op_cookie *cookie =
|
||||||
|
qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
|
||||||
|
vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
break;
|
||||||
|
enqueue_one_auth_job_gen1(ctx, req, &vec->digest[i],
|
||||||
|
&vec->auth_iv[i], ofs, (uint32_t)data_len);
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
|
||||||
|
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
|
||||||
|
qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
|
||||||
|
vec->src_sgl[i].num, NULL, &vec->auth_iv[i],
|
||||||
|
NULL, &vec->digest[i]);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
if (unlikely(i < n))
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue += i;
|
||||||
|
*status = 0;
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_dp_enqueue_single_chain_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
||||||
|
union rte_crypto_sym_ofs ofs,
|
||||||
|
struct rte_crypto_va_iova_ptr *cipher_iv,
|
||||||
|
struct rte_crypto_va_iova_ptr *digest,
|
||||||
|
struct rte_crypto_va_iova_ptr *auth_iv,
|
||||||
|
void *user_data)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_op_cookie *cookie;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
int32_t data_len;
|
||||||
|
uint32_t tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
cookie = qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
|
||||||
|
data, n_data_vecs, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
if (unlikely(enqueue_one_chain_job_gen1(ctx, req, data, n_data_vecs,
|
||||||
|
NULL, 0, cipher_iv, digest, auth_iv, ofs,
|
||||||
|
(uint32_t)data_len)))
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue++;
|
||||||
|
|
||||||
|
|
||||||
|
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
|
||||||
|
qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, cipher_iv,
|
||||||
|
auth_iv, NULL, digest);
|
||||||
|
#endif
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t
|
||||||
|
qat_sym_dp_enqueue_chain_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
||||||
|
void *user_data[], int *status)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
uint32_t i, n;
|
||||||
|
uint32_t tail;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
int32_t data_len;
|
||||||
|
|
||||||
|
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
|
||||||
|
if (unlikely(n == 0)) {
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
|
||||||
|
*status = 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
for (i = 0; i < n; i++) {
|
||||||
|
struct qat_sym_op_cookie *cookie =
|
||||||
|
qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
|
||||||
|
vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
break;
|
||||||
|
|
||||||
|
if (unlikely(enqueue_one_chain_job_gen1(ctx, req,
|
||||||
|
vec->src_sgl[i].vec, vec->src_sgl[i].num,
|
||||||
|
NULL, 0,
|
||||||
|
&vec->iv[i], &vec->digest[i],
|
||||||
|
&vec->auth_iv[i], ofs, (uint32_t)data_len)))
|
||||||
|
break;
|
||||||
|
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
|
||||||
|
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
|
||||||
|
qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
|
||||||
|
vec->src_sgl[i].num, &vec->iv[i],
|
||||||
|
&vec->auth_iv[i],
|
||||||
|
NULL, &vec->digest[i]);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
if (unlikely(i < n))
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue += i;
|
||||||
|
*status = 0;
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_dp_enqueue_single_aead_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
||||||
|
union rte_crypto_sym_ofs ofs,
|
||||||
|
struct rte_crypto_va_iova_ptr *iv,
|
||||||
|
struct rte_crypto_va_iova_ptr *digest,
|
||||||
|
struct rte_crypto_va_iova_ptr *aad,
|
||||||
|
void *user_data)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_op_cookie *cookie;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
|
||||||
|
int32_t data_len;
|
||||||
|
uint32_t tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
cookie = qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
|
||||||
|
data, n_data_vecs, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
enqueue_one_aead_job_gen1(ctx, req, iv, digest, aad, ofs,
|
||||||
|
(uint32_t)data_len);
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue++;
|
||||||
|
|
||||||
|
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
|
||||||
|
qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv,
|
||||||
|
NULL, aad, digest);
|
||||||
|
#endif
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t
|
||||||
|
qat_sym_dp_enqueue_aead_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
||||||
|
void *user_data[], int *status)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_session *ctx = dp_ctx->session;
|
||||||
|
uint32_t i, n;
|
||||||
|
uint32_t tail;
|
||||||
|
struct icp_qat_fw_la_bulk_req *req;
|
||||||
|
int32_t data_len;
|
||||||
|
|
||||||
|
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
|
||||||
|
if (unlikely(n == 0)) {
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
|
||||||
|
*status = 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
for (i = 0; i < n; i++) {
|
||||||
|
struct qat_sym_op_cookie *cookie =
|
||||||
|
qp->op_cookies[tail >> tx_queue->trailz];
|
||||||
|
|
||||||
|
req = (struct icp_qat_fw_la_bulk_req *)(
|
||||||
|
(uint8_t *)tx_queue->base_addr + tail);
|
||||||
|
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
||||||
|
|
||||||
|
data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
|
||||||
|
vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
|
||||||
|
if (unlikely(data_len < 0))
|
||||||
|
break;
|
||||||
|
|
||||||
|
enqueue_one_aead_job_gen1(ctx, req, &vec->iv[i],
|
||||||
|
&vec->digest[i], &vec->aad[i], ofs,
|
||||||
|
(uint32_t)data_len);
|
||||||
|
|
||||||
|
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
||||||
|
|
||||||
|
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
|
||||||
|
qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
|
||||||
|
vec->src_sgl[i].num, &vec->iv[i], NULL,
|
||||||
|
&vec->aad[i], &vec->digest[i]);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
if (unlikely(i < n))
|
||||||
|
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
|
||||||
|
|
||||||
|
dp_ctx->tail = tail;
|
||||||
|
dp_ctx->cached_enqueue += i;
|
||||||
|
*status = 0;
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
uint32_t
|
||||||
|
qat_sym_dp_dequeue_burst_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
|
||||||
|
uint32_t max_nb_to_dequeue,
|
||||||
|
rte_cryptodev_raw_post_dequeue_t post_dequeue,
|
||||||
|
void **out_user_data, uint8_t is_user_data_array,
|
||||||
|
uint32_t *n_success_jobs, int *return_status)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *rx_queue = &qp->rx_q;
|
||||||
|
struct icp_qat_fw_comn_resp *resp;
|
||||||
|
void *resp_opaque;
|
||||||
|
uint32_t i, n, inflight;
|
||||||
|
uint32_t head;
|
||||||
|
uint8_t status;
|
||||||
|
|
||||||
|
*n_success_jobs = 0;
|
||||||
|
*return_status = 0;
|
||||||
|
head = dp_ctx->head;
|
||||||
|
|
||||||
|
inflight = qp->enqueued - qp->dequeued;
|
||||||
|
if (unlikely(inflight == 0))
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +
|
||||||
|
head);
|
||||||
|
/* no operation ready */
|
||||||
|
if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
resp_opaque = (void *)(uintptr_t)resp->opaque_data;
|
||||||
|
/* get the dequeue count */
|
||||||
|
if (get_dequeue_count) {
|
||||||
|
n = get_dequeue_count(resp_opaque);
|
||||||
|
if (unlikely(n == 0))
|
||||||
|
return 0;
|
||||||
|
} else {
|
||||||
|
if (unlikely(max_nb_to_dequeue == 0))
|
||||||
|
return 0;
|
||||||
|
n = max_nb_to_dequeue;
|
||||||
|
}
|
||||||
|
|
||||||
|
out_user_data[0] = resp_opaque;
|
||||||
|
status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
|
||||||
|
post_dequeue(resp_opaque, 0, status);
|
||||||
|
*n_success_jobs += status;
|
||||||
|
|
||||||
|
head = (head + rx_queue->msg_size) & rx_queue->modulo_mask;
|
||||||
|
|
||||||
|
/* we already finished dequeue when n == 1 */
|
||||||
|
if (unlikely(n == 1)) {
|
||||||
|
i = 1;
|
||||||
|
goto end_deq;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (is_user_data_array) {
|
||||||
|
for (i = 1; i < n; i++) {
|
||||||
|
resp = (struct icp_qat_fw_comn_resp *)(
|
||||||
|
(uint8_t *)rx_queue->base_addr + head);
|
||||||
|
if (unlikely(*(uint32_t *)resp ==
|
||||||
|
ADF_RING_EMPTY_SIG))
|
||||||
|
goto end_deq;
|
||||||
|
out_user_data[i] = (void *)(uintptr_t)resp->opaque_data;
|
||||||
|
status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
|
||||||
|
*n_success_jobs += status;
|
||||||
|
post_dequeue(out_user_data[i], i, status);
|
||||||
|
head = (head + rx_queue->msg_size) &
|
||||||
|
rx_queue->modulo_mask;
|
||||||
|
}
|
||||||
|
|
||||||
|
goto end_deq;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* opaque is not array */
|
||||||
|
for (i = 1; i < n; i++) {
|
||||||
|
resp = (struct icp_qat_fw_comn_resp *)(
|
||||||
|
(uint8_t *)rx_queue->base_addr + head);
|
||||||
|
status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
|
||||||
|
if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
|
||||||
|
goto end_deq;
|
||||||
|
head = (head + rx_queue->msg_size) &
|
||||||
|
rx_queue->modulo_mask;
|
||||||
|
post_dequeue(resp_opaque, i, status);
|
||||||
|
*n_success_jobs += status;
|
||||||
|
}
|
||||||
|
|
||||||
|
end_deq:
|
||||||
|
dp_ctx->head = head;
|
||||||
|
dp_ctx->cached_dequeue += i;
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
|
||||||
|
void *
|
||||||
|
qat_sym_dp_dequeue_single_gen1(void *qp_data, uint8_t *drv_ctx,
|
||||||
|
int *dequeue_status, enum rte_crypto_op_status *op_status)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
struct qat_queue *rx_queue = &qp->rx_q;
|
||||||
|
register struct icp_qat_fw_comn_resp *resp;
|
||||||
|
|
||||||
|
resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +
|
||||||
|
dp_ctx->head);
|
||||||
|
|
||||||
|
if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
dp_ctx->head = (dp_ctx->head + rx_queue->msg_size) &
|
||||||
|
rx_queue->modulo_mask;
|
||||||
|
dp_ctx->cached_dequeue++;
|
||||||
|
|
||||||
|
*op_status = QAT_SYM_DP_IS_RESP_SUCCESS(resp) ?
|
||||||
|
RTE_CRYPTO_OP_STATUS_SUCCESS :
|
||||||
|
RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
|
||||||
|
*dequeue_status = 0;
|
||||||
|
return (void *)(uintptr_t)resp->opaque_data;
|
||||||
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_queue *tx_queue = &qp->tx_q;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
|
||||||
|
if (unlikely(dp_ctx->cached_enqueue != n))
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
qp->enqueued += n;
|
||||||
|
qp->stats.enqueued_count += n;
|
||||||
|
|
||||||
|
tx_queue->tail = dp_ctx->tail;
|
||||||
|
|
||||||
|
WRITE_CSR_RING_TAIL(qp->mmap_bar_addr,
|
||||||
|
tx_queue->hw_bundle_number,
|
||||||
|
tx_queue->hw_queue_number, tx_queue->tail);
|
||||||
|
tx_queue->csr_tail = tx_queue->tail;
|
||||||
|
dp_ctx->cached_enqueue = 0;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_dp_dequeue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n)
|
||||||
|
{
|
||||||
|
struct qat_qp *qp = qp_data;
|
||||||
|
struct qat_queue *rx_queue = &qp->rx_q;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
||||||
|
|
||||||
|
if (unlikely(dp_ctx->cached_dequeue != n))
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
rx_queue->head = dp_ctx->head;
|
||||||
|
rx_queue->nb_processed_responses += n;
|
||||||
|
qp->dequeued += n;
|
||||||
|
qp->stats.dequeued_count += n;
|
||||||
|
if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) {
|
||||||
|
uint32_t old_head, new_head;
|
||||||
|
uint32_t max_head;
|
||||||
|
|
||||||
|
old_head = rx_queue->csr_head;
|
||||||
|
new_head = rx_queue->head;
|
||||||
|
max_head = qp->nb_descriptors * rx_queue->msg_size;
|
||||||
|
|
||||||
|
/* write out free descriptors */
|
||||||
|
void *cur_desc = (uint8_t *)rx_queue->base_addr + old_head;
|
||||||
|
|
||||||
|
if (new_head < old_head) {
|
||||||
|
memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE,
|
||||||
|
max_head - old_head);
|
||||||
|
memset(rx_queue->base_addr, ADF_RING_EMPTY_SIG_BYTE,
|
||||||
|
new_head);
|
||||||
|
} else {
|
||||||
|
memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head -
|
||||||
|
old_head);
|
||||||
|
}
|
||||||
|
rx_queue->nb_processed_responses = 0;
|
||||||
|
rx_queue->csr_head = new_head;
|
||||||
|
|
||||||
|
/* write current head to CSR */
|
||||||
|
WRITE_CSR_RING_HEAD(qp->mmap_bar_addr,
|
||||||
|
rx_queue->hw_bundle_number, rx_queue->hw_queue_number,
|
||||||
|
new_head);
|
||||||
|
}
|
||||||
|
|
||||||
|
dp_ctx->cached_dequeue = 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx)
|
||||||
|
{
|
||||||
|
struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;
|
||||||
|
struct qat_sym_session *ctx = _ctx;
|
||||||
|
|
||||||
|
raw_dp_ctx->enqueue_done = qat_sym_dp_enqueue_done_gen1;
|
||||||
|
raw_dp_ctx->dequeue_burst = qat_sym_dp_dequeue_burst_gen1;
|
||||||
|
raw_dp_ctx->dequeue = qat_sym_dp_dequeue_single_gen1;
|
||||||
|
raw_dp_ctx->dequeue_done = qat_sym_dp_dequeue_done_gen1;
|
||||||
|
|
||||||
|
if ((ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
|
||||||
|
ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) &&
|
||||||
|
!ctx->is_gmac) {
|
||||||
|
/* AES-GCM or AES-CCM */
|
||||||
|
if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
|
||||||
|
ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||
|
||||||
|
(ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128
|
||||||
|
&& ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE
|
||||||
|
&& ctx->qat_hash_alg ==
|
||||||
|
ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {
|
||||||
|
raw_dp_ctx->enqueue_burst =
|
||||||
|
qat_sym_dp_enqueue_aead_jobs_gen1;
|
||||||
|
raw_dp_ctx->enqueue =
|
||||||
|
qat_sym_dp_enqueue_single_aead_gen1;
|
||||||
|
} else {
|
||||||
|
raw_dp_ctx->enqueue_burst =
|
||||||
|
qat_sym_dp_enqueue_chain_jobs_gen1;
|
||||||
|
raw_dp_ctx->enqueue =
|
||||||
|
qat_sym_dp_enqueue_single_chain_gen1;
|
||||||
|
}
|
||||||
|
} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH || ctx->is_gmac) {
|
||||||
|
raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs_gen1;
|
||||||
|
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth_gen1;
|
||||||
|
} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
|
||||||
|
if (ctx->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE ||
|
||||||
|
ctx->qat_cipher_alg ==
|
||||||
|
ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305) {
|
||||||
|
raw_dp_ctx->enqueue_burst =
|
||||||
|
qat_sym_dp_enqueue_aead_jobs_gen1;
|
||||||
|
raw_dp_ctx->enqueue =
|
||||||
|
qat_sym_dp_enqueue_single_aead_gen1;
|
||||||
|
} else {
|
||||||
|
raw_dp_ctx->enqueue_burst =
|
||||||
|
qat_sym_dp_enqueue_cipher_jobs_gen1;
|
||||||
|
raw_dp_ctx->enqueue =
|
||||||
|
qat_sym_dp_enqueue_single_cipher_gen1;
|
||||||
|
}
|
||||||
|
} else
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
int
|
int
|
||||||
qat_sym_crypto_set_session_gen1(void *cryptodev __rte_unused, void *session)
|
qat_sym_crypto_set_session_gen1(void *cryptodev __rte_unused, void *session)
|
||||||
{
|
{
|
||||||
@ -518,6 +1172,8 @@ RTE_INIT(qat_sym_crypto_gen1_init)
|
|||||||
qat_sym_crypto_cap_get_gen1;
|
qat_sym_crypto_cap_get_gen1;
|
||||||
qat_sym_gen_dev_ops[QAT_GEN1].set_session =
|
qat_sym_gen_dev_ops[QAT_GEN1].set_session =
|
||||||
qat_sym_crypto_set_session_gen1;
|
qat_sym_crypto_set_session_gen1;
|
||||||
|
qat_sym_gen_dev_ops[QAT_GEN1].set_raw_dp_ctx =
|
||||||
|
qat_sym_configure_raw_dp_ctx_gen1;
|
||||||
qat_sym_gen_dev_ops[QAT_GEN1].get_feature_flags =
|
qat_sym_gen_dev_ops[QAT_GEN1].get_feature_flags =
|
||||||
qat_sym_crypto_feature_flags_get_gen1;
|
qat_sym_crypto_feature_flags_get_gen1;
|
||||||
#ifdef RTE_LIB_SECURITY
|
#ifdef RTE_LIB_SECURITY
|
||||||
|
|||||||
@ -53,11 +53,14 @@ typedef void * (*create_security_ctx_t)(void *cryptodev);
|
|||||||
|
|
||||||
typedef int (*set_session_t)(void *cryptodev, void *session);
|
typedef int (*set_session_t)(void *cryptodev, void *session);
|
||||||
|
|
||||||
|
typedef int (*set_raw_dp_ctx_t)(void *raw_dp_ctx, void *ctx);
|
||||||
|
|
||||||
struct qat_crypto_gen_dev_ops {
|
struct qat_crypto_gen_dev_ops {
|
||||||
get_feature_flags_t get_feature_flags;
|
get_feature_flags_t get_feature_flags;
|
||||||
get_capabilities_info_t get_capabilities;
|
get_capabilities_info_t get_capabilities;
|
||||||
struct rte_cryptodev_ops *cryptodev_ops;
|
struct rte_cryptodev_ops *cryptodev_ops;
|
||||||
set_session_t set_session;
|
set_session_t set_session;
|
||||||
|
set_raw_dp_ctx_t set_raw_dp_ctx;
|
||||||
#ifdef RTE_LIB_SECURITY
|
#ifdef RTE_LIB_SECURITY
|
||||||
create_security_ctx_t create_security_ctx;
|
create_security_ctx_t create_security_ctx;
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@ -89,7 +89,7 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
|
|||||||
(void *)cdev, (void *)sess);
|
(void *)cdev, (void *)sess);
|
||||||
if (ret < 0) {
|
if (ret < 0) {
|
||||||
op->status =
|
op->status =
|
||||||
RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
|
RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -144,7 +144,7 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
|
|||||||
(void *)cdev, (void *)sess);
|
(void *)cdev, (void *)sess);
|
||||||
if (ret < 0) {
|
if (ret < 0) {
|
||||||
op->status =
|
op->status =
|
||||||
RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
|
RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
|
||||||
return -EINVAL;
|
return -EINVAL;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -292,8 +292,7 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
|
|||||||
if (internals->capa_mz == NULL) {
|
if (internals->capa_mz == NULL) {
|
||||||
QAT_LOG(DEBUG,
|
QAT_LOG(DEBUG,
|
||||||
"Error allocating memzone for capabilities, "
|
"Error allocating memzone for capabilities, "
|
||||||
"destroying PMD for %s",
|
"destroying PMD for %s", name);
|
||||||
name);
|
|
||||||
ret = -EFAULT;
|
ret = -EFAULT;
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
@ -355,6 +354,55 @@ qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev)
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
|
||||||
|
struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
|
||||||
|
enum rte_crypto_op_sess_type sess_type,
|
||||||
|
union rte_cryptodev_session_ctx session_ctx, uint8_t is_update)
|
||||||
|
{
|
||||||
|
struct qat_cryptodev_private *internals = dev->data->dev_private;
|
||||||
|
enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen;
|
||||||
|
struct qat_crypto_gen_dev_ops *gen_dev_ops =
|
||||||
|
&qat_sym_gen_dev_ops[qat_dev_gen];
|
||||||
|
struct qat_qp *qp;
|
||||||
|
struct qat_sym_session *ctx;
|
||||||
|
struct qat_sym_dp_ctx *dp_ctx;
|
||||||
|
|
||||||
|
if (!gen_dev_ops->set_raw_dp_ctx) {
|
||||||
|
QAT_LOG(ERR, "Device GEN %u does not support raw data path",
|
||||||
|
qat_dev_gen);
|
||||||
|
return -ENOTSUP;
|
||||||
|
}
|
||||||
|
|
||||||
|
qp = dev->data->queue_pairs[qp_id];
|
||||||
|
dp_ctx = (struct qat_sym_dp_ctx *)raw_dp_ctx->drv_ctx_data;
|
||||||
|
|
||||||
|
if (!is_update) {
|
||||||
|
memset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx) +
|
||||||
|
sizeof(struct qat_sym_dp_ctx));
|
||||||
|
raw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];
|
||||||
|
dp_ctx->tail = qp->tx_q.tail;
|
||||||
|
dp_ctx->head = qp->rx_q.head;
|
||||||
|
dp_ctx->cached_enqueue = dp_ctx->cached_dequeue = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (sess_type != RTE_CRYPTO_OP_WITH_SESSION)
|
||||||
|
return -EINVAL;
|
||||||
|
|
||||||
|
ctx = (struct qat_sym_session *)get_sym_session_private_data(
|
||||||
|
session_ctx.crypto_sess, qat_sym_driver_id);
|
||||||
|
|
||||||
|
dp_ctx->session = ctx;
|
||||||
|
|
||||||
|
return gen_dev_ops->set_raw_dp_ctx(raw_dp_ctx, ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
qat_sym_get_dp_ctx_size(struct rte_cryptodev *dev __rte_unused)
|
||||||
|
{
|
||||||
|
return sizeof(struct qat_sym_dp_ctx);
|
||||||
|
}
|
||||||
|
|
||||||
static struct cryptodev_driver qat_crypto_drv;
|
static struct cryptodev_driver qat_crypto_drv;
|
||||||
RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
|
RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
|
||||||
cryptodev_qat_sym_driver,
|
cryptodev_qat_sym_driver,
|
||||||
|
|||||||
@ -1,986 +0,0 @@
|
|||||||
/* SPDX-License-Identifier: BSD-3-Clause
|
|
||||||
* Copyright(c) 2022 Intel Corporation
|
|
||||||
*/
|
|
||||||
|
|
||||||
#include <cryptodev_pmd.h>
|
|
||||||
|
|
||||||
#include "adf_transport_access_macros.h"
|
|
||||||
#include "icp_qat_fw.h"
|
|
||||||
#include "icp_qat_fw_la.h"
|
|
||||||
|
|
||||||
#include "qat_sym.h"
|
|
||||||
#include "qat_sym_session.h"
|
|
||||||
#include "qat_qp.h"
|
|
||||||
|
|
||||||
static __rte_always_inline int32_t
|
|
||||||
qat_sym_dp_parse_data_vec(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req,
|
|
||||||
struct rte_crypto_vec *data, uint16_t n_data_vecs)
|
|
||||||
{
|
|
||||||
struct qat_queue *tx_queue;
|
|
||||||
struct qat_sym_op_cookie *cookie;
|
|
||||||
struct qat_sgl *list;
|
|
||||||
uint32_t i;
|
|
||||||
uint32_t total_len;
|
|
||||||
|
|
||||||
if (likely(n_data_vecs == 1)) {
|
|
||||||
req->comn_mid.src_data_addr = req->comn_mid.dest_data_addr =
|
|
||||||
data[0].iova;
|
|
||||||
req->comn_mid.src_length = req->comn_mid.dst_length =
|
|
||||||
data[0].len;
|
|
||||||
return data[0].len;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (n_data_vecs == 0 || n_data_vecs > QAT_SYM_SGL_MAX_NUMBER)
|
|
||||||
return -1;
|
|
||||||
|
|
||||||
total_len = 0;
|
|
||||||
tx_queue = &qp->tx_q;
|
|
||||||
|
|
||||||
ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
|
|
||||||
QAT_COMN_PTR_TYPE_SGL);
|
|
||||||
cookie = qp->op_cookies[tx_queue->tail >> tx_queue->trailz];
|
|
||||||
list = (struct qat_sgl *)&cookie->qat_sgl_src;
|
|
||||||
|
|
||||||
for (i = 0; i < n_data_vecs; i++) {
|
|
||||||
list->buffers[i].len = data[i].len;
|
|
||||||
list->buffers[i].resrvd = 0;
|
|
||||||
list->buffers[i].addr = data[i].iova;
|
|
||||||
if (total_len + data[i].len > UINT32_MAX) {
|
|
||||||
QAT_DP_LOG(ERR, "Message too long");
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
total_len += data[i].len;
|
|
||||||
}
|
|
||||||
|
|
||||||
list->num_bufs = i;
|
|
||||||
req->comn_mid.src_data_addr = req->comn_mid.dest_data_addr =
|
|
||||||
cookie->qat_sgl_src_phys_addr;
|
|
||||||
req->comn_mid.src_length = req->comn_mid.dst_length = 0;
|
|
||||||
return total_len;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline void
|
|
||||||
set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
|
|
||||||
struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
|
|
||||||
struct icp_qat_fw_la_bulk_req *qat_req)
|
|
||||||
{
|
|
||||||
/* copy IV into request if it fits */
|
|
||||||
if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
|
|
||||||
rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
|
|
||||||
iv_len);
|
|
||||||
else {
|
|
||||||
ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
|
|
||||||
qat_req->comn_hdr.serv_specif_flags,
|
|
||||||
ICP_QAT_FW_CIPH_IV_64BIT_PTR);
|
|
||||||
cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
|
|
||||||
(ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
|
|
||||||
ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
|
|
||||||
|
|
||||||
static __rte_always_inline void
|
|
||||||
qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
|
|
||||||
{
|
|
||||||
uint32_t i;
|
|
||||||
|
|
||||||
for (i = 0; i < n; i++)
|
|
||||||
sta[i] = status;
|
|
||||||
}
|
|
||||||
|
|
||||||
#define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
|
|
||||||
RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
|
|
||||||
|
|
||||||
static __rte_always_inline void
|
|
||||||
enqueue_one_cipher_job(struct qat_sym_session *ctx,
|
|
||||||
struct icp_qat_fw_la_bulk_req *req,
|
|
||||||
struct rte_crypto_va_iova_ptr *iv,
|
|
||||||
union rte_crypto_sym_ofs ofs, uint32_t data_len)
|
|
||||||
{
|
|
||||||
struct icp_qat_fw_la_cipher_req_params *cipher_param;
|
|
||||||
|
|
||||||
cipher_param = (void *)&req->serv_specif_rqpars;
|
|
||||||
|
|
||||||
/* cipher IV */
|
|
||||||
set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
|
|
||||||
cipher_param->cipher_offset = ofs.ofs.cipher.head;
|
|
||||||
cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
|
|
||||||
ofs.ofs.cipher.tail;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline int
|
|
||||||
qat_sym_dp_enqueue_single_cipher(void *qp_data, uint8_t *drv_ctx,
|
|
||||||
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
|
||||||
union rte_crypto_sym_ofs ofs,
|
|
||||||
struct rte_crypto_va_iova_ptr *iv,
|
|
||||||
struct rte_crypto_va_iova_ptr *digest __rte_unused,
|
|
||||||
struct rte_crypto_va_iova_ptr *aad __rte_unused,
|
|
||||||
void *user_data)
|
|
||||||
{
|
|
||||||
struct qat_qp *qp = qp_data;
|
|
||||||
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
|
||||||
struct qat_queue *tx_queue = &qp->tx_q;
|
|
||||||
struct qat_sym_session *ctx = dp_ctx->session;
|
|
||||||
struct icp_qat_fw_la_bulk_req *req;
|
|
||||||
int32_t data_len;
|
|
||||||
uint32_t tail = dp_ctx->tail;
|
|
||||||
|
|
||||||
req = (struct icp_qat_fw_la_bulk_req *)(
|
|
||||||
(uint8_t *)tx_queue->base_addr + tail);
|
|
||||||
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
|
||||||
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
|
||||||
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
|
|
||||||
data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
|
|
||||||
if (unlikely(data_len < 0))
|
|
||||||
return -1;
|
|
||||||
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
|
|
||||||
|
|
||||||
enqueue_one_cipher_job(ctx, req, iv, ofs, (uint32_t)data_len);
|
|
||||||
|
|
||||||
dp_ctx->tail = tail;
|
|
||||||
dp_ctx->cached_enqueue++;
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline uint32_t
|
|
||||||
qat_sym_dp_enqueue_cipher_jobs(void *qp_data, uint8_t *drv_ctx,
|
|
||||||
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
|
||||||
void *user_data[], int *status)
|
|
||||||
{
|
|
||||||
struct qat_qp *qp = qp_data;
|
|
||||||
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
|
||||||
struct qat_queue *tx_queue = &qp->tx_q;
|
|
||||||
struct qat_sym_session *ctx = dp_ctx->session;
|
|
||||||
uint32_t i, n;
|
|
||||||
uint32_t tail;
|
|
||||||
struct icp_qat_fw_la_bulk_req *req;
|
|
||||||
int32_t data_len;
|
|
||||||
|
|
||||||
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
|
|
||||||
if (unlikely(n == 0)) {
|
|
||||||
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
|
|
||||||
*status = 0;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
tail = dp_ctx->tail;
|
|
||||||
|
|
||||||
for (i = 0; i < n; i++) {
|
|
||||||
req = (struct icp_qat_fw_la_bulk_req *)(
|
|
||||||
(uint8_t *)tx_queue->base_addr + tail);
|
|
||||||
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
|
||||||
|
|
||||||
data_len = qat_sym_dp_parse_data_vec(qp, req,
|
|
||||||
vec->src_sgl[i].vec,
|
|
||||||
vec->src_sgl[i].num);
|
|
||||||
if (unlikely(data_len < 0))
|
|
||||||
break;
|
|
||||||
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
|
|
||||||
enqueue_one_cipher_job(ctx, req, &vec->iv[i], ofs,
|
|
||||||
(uint32_t)data_len);
|
|
||||||
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (unlikely(i < n))
|
|
||||||
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
|
|
||||||
|
|
||||||
dp_ctx->tail = tail;
|
|
||||||
dp_ctx->cached_enqueue += i;
|
|
||||||
*status = 0;
|
|
||||||
return i;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline void
|
|
||||||
enqueue_one_auth_job(struct qat_sym_session *ctx,
|
|
||||||
struct icp_qat_fw_la_bulk_req *req,
|
|
||||||
struct rte_crypto_va_iova_ptr *digest,
|
|
||||||
struct rte_crypto_va_iova_ptr *auth_iv,
|
|
||||||
union rte_crypto_sym_ofs ofs, uint32_t data_len)
|
|
||||||
{
|
|
||||||
struct icp_qat_fw_la_cipher_req_params *cipher_param;
|
|
||||||
struct icp_qat_fw_la_auth_req_params *auth_param;
|
|
||||||
|
|
||||||
cipher_param = (void *)&req->serv_specif_rqpars;
|
|
||||||
auth_param = (void *)((uint8_t *)cipher_param +
|
|
||||||
ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
|
|
||||||
|
|
||||||
auth_param->auth_off = ofs.ofs.auth.head;
|
|
||||||
auth_param->auth_len = data_len - ofs.ofs.auth.head -
|
|
||||||
ofs.ofs.auth.tail;
|
|
||||||
auth_param->auth_res_addr = digest->iova;
|
|
||||||
|
|
||||||
switch (ctx->qat_hash_alg) {
|
|
||||||
case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
|
|
||||||
case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
|
|
||||||
case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
|
|
||||||
auth_param->u1.aad_adr = auth_iv->iova;
|
|
||||||
break;
|
|
||||||
case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
|
|
||||||
case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
|
|
||||||
ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
|
|
||||||
req->comn_hdr.serv_specif_flags,
|
|
||||||
ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
|
|
||||||
rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
|
|
||||||
ctx->auth_iv.length);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline int
|
|
||||||
qat_sym_dp_enqueue_single_auth(void *qp_data, uint8_t *drv_ctx,
|
|
||||||
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
|
||||||
union rte_crypto_sym_ofs ofs,
|
|
||||||
struct rte_crypto_va_iova_ptr *iv __rte_unused,
|
|
||||||
struct rte_crypto_va_iova_ptr *digest,
|
|
||||||
struct rte_crypto_va_iova_ptr *auth_iv,
|
|
||||||
void *user_data)
|
|
||||||
{
|
|
||||||
struct qat_qp *qp = qp_data;
|
|
||||||
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
|
||||||
struct qat_queue *tx_queue = &qp->tx_q;
|
|
||||||
struct qat_sym_session *ctx = dp_ctx->session;
|
|
||||||
struct icp_qat_fw_la_bulk_req *req;
|
|
||||||
int32_t data_len;
|
|
||||||
uint32_t tail = dp_ctx->tail;
|
|
||||||
|
|
||||||
req = (struct icp_qat_fw_la_bulk_req *)(
|
|
||||||
(uint8_t *)tx_queue->base_addr + tail);
|
|
||||||
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
|
||||||
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
|
||||||
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
|
|
||||||
data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
|
|
||||||
if (unlikely(data_len < 0))
|
|
||||||
return -1;
|
|
||||||
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
|
|
||||||
|
|
||||||
enqueue_one_auth_job(ctx, req, digest, auth_iv, ofs,
|
|
||||||
(uint32_t)data_len);
|
|
||||||
|
|
||||||
dp_ctx->tail = tail;
|
|
||||||
dp_ctx->cached_enqueue++;
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline uint32_t
|
|
||||||
qat_sym_dp_enqueue_auth_jobs(void *qp_data, uint8_t *drv_ctx,
|
|
||||||
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
|
||||||
void *user_data[], int *status)
|
|
||||||
{
|
|
||||||
struct qat_qp *qp = qp_data;
|
|
||||||
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
|
||||||
struct qat_queue *tx_queue = &qp->tx_q;
|
|
||||||
struct qat_sym_session *ctx = dp_ctx->session;
|
|
||||||
uint32_t i, n;
|
|
||||||
uint32_t tail;
|
|
||||||
struct icp_qat_fw_la_bulk_req *req;
|
|
||||||
int32_t data_len;
|
|
||||||
|
|
||||||
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
|
|
||||||
if (unlikely(n == 0)) {
|
|
||||||
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
|
|
||||||
*status = 0;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
tail = dp_ctx->tail;
|
|
||||||
|
|
||||||
for (i = 0; i < n; i++) {
|
|
||||||
req = (struct icp_qat_fw_la_bulk_req *)(
|
|
||||||
(uint8_t *)tx_queue->base_addr + tail);
|
|
||||||
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
|
||||||
|
|
||||||
data_len = qat_sym_dp_parse_data_vec(qp, req,
|
|
||||||
vec->src_sgl[i].vec,
|
|
||||||
vec->src_sgl[i].num);
|
|
||||||
if (unlikely(data_len < 0))
|
|
||||||
break;
|
|
||||||
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
|
|
||||||
enqueue_one_auth_job(ctx, req, &vec->digest[i],
|
|
||||||
&vec->auth_iv[i], ofs, (uint32_t)data_len);
|
|
||||||
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (unlikely(i < n))
|
|
||||||
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
|
|
||||||
|
|
||||||
dp_ctx->tail = tail;
|
|
||||||
dp_ctx->cached_enqueue += i;
|
|
||||||
*status = 0;
|
|
||||||
return i;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline int
|
|
||||||
enqueue_one_chain_job(struct qat_sym_session *ctx,
|
|
||||||
struct icp_qat_fw_la_bulk_req *req,
|
|
||||||
struct rte_crypto_vec *data,
|
|
||||||
uint16_t n_data_vecs,
|
|
||||||
struct rte_crypto_va_iova_ptr *cipher_iv,
|
|
||||||
struct rte_crypto_va_iova_ptr *digest,
|
|
||||||
struct rte_crypto_va_iova_ptr *auth_iv,
|
|
||||||
union rte_crypto_sym_ofs ofs, uint32_t data_len)
|
|
||||||
{
|
|
||||||
struct icp_qat_fw_la_cipher_req_params *cipher_param;
|
|
||||||
struct icp_qat_fw_la_auth_req_params *auth_param;
|
|
||||||
rte_iova_t auth_iova_end;
|
|
||||||
int32_t cipher_len, auth_len;
|
|
||||||
|
|
||||||
cipher_param = (void *)&req->serv_specif_rqpars;
|
|
||||||
auth_param = (void *)((uint8_t *)cipher_param +
|
|
||||||
ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
|
|
||||||
|
|
||||||
cipher_len = data_len - ofs.ofs.cipher.head -
|
|
||||||
ofs.ofs.cipher.tail;
|
|
||||||
auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
|
|
||||||
|
|
||||||
if (unlikely(cipher_len < 0 || auth_len < 0))
|
|
||||||
return -1;
|
|
||||||
|
|
||||||
cipher_param->cipher_offset = ofs.ofs.cipher.head;
|
|
||||||
cipher_param->cipher_length = cipher_len;
|
|
||||||
set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
|
|
||||||
|
|
||||||
auth_param->auth_off = ofs.ofs.auth.head;
|
|
||||||
auth_param->auth_len = auth_len;
|
|
||||||
auth_param->auth_res_addr = digest->iova;
|
|
||||||
|
|
||||||
switch (ctx->qat_hash_alg) {
|
|
||||||
case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
|
|
||||||
case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
|
|
||||||
case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
|
|
||||||
auth_param->u1.aad_adr = auth_iv->iova;
|
|
||||||
break;
|
|
||||||
case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
|
|
||||||
case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (unlikely(n_data_vecs > 1)) {
|
|
||||||
int auth_end_get = 0, i = n_data_vecs - 1;
|
|
||||||
struct rte_crypto_vec *cvec = &data[0];
|
|
||||||
uint32_t len;
|
|
||||||
|
|
||||||
len = data_len - ofs.ofs.auth.tail;
|
|
||||||
|
|
||||||
while (i >= 0 && len > 0) {
|
|
||||||
if (cvec->len >= len) {
|
|
||||||
auth_iova_end = cvec->iova + len;
|
|
||||||
len = 0;
|
|
||||||
auth_end_get = 1;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
len -= cvec->len;
|
|
||||||
i--;
|
|
||||||
cvec++;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (unlikely(auth_end_get == 0))
|
|
||||||
return -1;
|
|
||||||
} else
|
|
||||||
auth_iova_end = data[0].iova + auth_param->auth_off +
|
|
||||||
auth_param->auth_len;
|
|
||||||
|
|
||||||
/* Then check if digest-encrypted conditions are met */
|
|
||||||
if ((auth_param->auth_off + auth_param->auth_len <
|
|
||||||
cipher_param->cipher_offset +
|
|
||||||
cipher_param->cipher_length) &&
|
|
||||||
(digest->iova == auth_iova_end)) {
|
|
||||||
/* Handle partial digest encryption */
|
|
||||||
if (cipher_param->cipher_offset +
|
|
||||||
cipher_param->cipher_length <
|
|
||||||
auth_param->auth_off +
|
|
||||||
auth_param->auth_len +
|
|
||||||
ctx->digest_length)
|
|
||||||
req->comn_mid.dst_length =
|
|
||||||
req->comn_mid.src_length =
|
|
||||||
auth_param->auth_off +
|
|
||||||
auth_param->auth_len +
|
|
||||||
ctx->digest_length;
|
|
||||||
struct icp_qat_fw_comn_req_hdr *header =
|
|
||||||
&req->comn_hdr;
|
|
||||||
ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
|
|
||||||
header->serv_specif_flags,
|
|
||||||
ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline int
|
|
||||||
qat_sym_dp_enqueue_single_chain(void *qp_data, uint8_t *drv_ctx,
|
|
||||||
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
|
||||||
union rte_crypto_sym_ofs ofs,
|
|
||||||
struct rte_crypto_va_iova_ptr *cipher_iv,
|
|
||||||
struct rte_crypto_va_iova_ptr *digest,
|
|
||||||
struct rte_crypto_va_iova_ptr *auth_iv,
|
|
||||||
void *user_data)
|
|
||||||
{
|
|
||||||
struct qat_qp *qp = qp_data;
|
|
||||||
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
|
||||||
struct qat_queue *tx_queue = &qp->tx_q;
|
|
||||||
struct qat_sym_session *ctx = dp_ctx->session;
|
|
||||||
struct icp_qat_fw_la_bulk_req *req;
|
|
||||||
int32_t data_len;
|
|
||||||
uint32_t tail = dp_ctx->tail;
|
|
||||||
|
|
||||||
req = (struct icp_qat_fw_la_bulk_req *)(
|
|
||||||
(uint8_t *)tx_queue->base_addr + tail);
|
|
||||||
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
|
||||||
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
|
||||||
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
|
|
||||||
data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
|
|
||||||
if (unlikely(data_len < 0))
|
|
||||||
return -1;
|
|
||||||
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
|
|
||||||
|
|
||||||
if (unlikely(enqueue_one_chain_job(ctx, req, data, n_data_vecs,
|
|
||||||
cipher_iv, digest, auth_iv, ofs, (uint32_t)data_len)))
|
|
||||||
return -1;
|
|
||||||
|
|
||||||
dp_ctx->tail = tail;
|
|
||||||
dp_ctx->cached_enqueue++;
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline uint32_t
|
|
||||||
qat_sym_dp_enqueue_chain_jobs(void *qp_data, uint8_t *drv_ctx,
|
|
||||||
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
|
||||||
void *user_data[], int *status)
|
|
||||||
{
|
|
||||||
struct qat_qp *qp = qp_data;
|
|
||||||
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
|
||||||
struct qat_queue *tx_queue = &qp->tx_q;
|
|
||||||
struct qat_sym_session *ctx = dp_ctx->session;
|
|
||||||
uint32_t i, n;
|
|
||||||
uint32_t tail;
|
|
||||||
struct icp_qat_fw_la_bulk_req *req;
|
|
||||||
int32_t data_len;
|
|
||||||
|
|
||||||
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
|
|
||||||
if (unlikely(n == 0)) {
|
|
||||||
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
|
|
||||||
*status = 0;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
tail = dp_ctx->tail;
|
|
||||||
|
|
||||||
for (i = 0; i < n; i++) {
|
|
||||||
req = (struct icp_qat_fw_la_bulk_req *)(
|
|
||||||
(uint8_t *)tx_queue->base_addr + tail);
|
|
||||||
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
|
||||||
|
|
||||||
data_len = qat_sym_dp_parse_data_vec(qp, req,
|
|
||||||
vec->src_sgl[i].vec,
|
|
||||||
vec->src_sgl[i].num);
|
|
||||||
if (unlikely(data_len < 0))
|
|
||||||
break;
|
|
||||||
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
|
|
||||||
if (unlikely(enqueue_one_chain_job(ctx, req,
|
|
||||||
vec->src_sgl[i].vec, vec->src_sgl[i].num,
|
|
||||||
&vec->iv[i], &vec->digest[i],
|
|
||||||
&vec->auth_iv[i], ofs, (uint32_t)data_len)))
|
|
||||||
break;
|
|
||||||
|
|
||||||
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (unlikely(i < n))
|
|
||||||
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
|
|
||||||
|
|
||||||
dp_ctx->tail = tail;
|
|
||||||
dp_ctx->cached_enqueue += i;
|
|
||||||
*status = 0;
|
|
||||||
return i;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline void
|
|
||||||
enqueue_one_aead_job(struct qat_sym_session *ctx,
|
|
||||||
struct icp_qat_fw_la_bulk_req *req,
|
|
||||||
struct rte_crypto_va_iova_ptr *iv,
|
|
||||||
struct rte_crypto_va_iova_ptr *digest,
|
|
||||||
struct rte_crypto_va_iova_ptr *aad,
|
|
||||||
union rte_crypto_sym_ofs ofs, uint32_t data_len)
|
|
||||||
{
|
|
||||||
struct icp_qat_fw_la_cipher_req_params *cipher_param =
|
|
||||||
(void *)&req->serv_specif_rqpars;
|
|
||||||
struct icp_qat_fw_la_auth_req_params *auth_param =
|
|
||||||
(void *)((uint8_t *)&req->serv_specif_rqpars +
|
|
||||||
ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
|
|
||||||
uint8_t *aad_data;
|
|
||||||
uint8_t aad_ccm_real_len;
|
|
||||||
uint8_t aad_len_field_sz;
|
|
||||||
uint32_t msg_len_be;
|
|
||||||
rte_iova_t aad_iova = 0;
|
|
||||||
uint8_t q;
|
|
||||||
|
|
||||||
/* CPM 1.7 uses single pass to treat AEAD as cipher operation */
|
|
||||||
if (ctx->is_single_pass) {
|
|
||||||
enqueue_one_cipher_job(ctx, req, iv, ofs, data_len);
|
|
||||||
|
|
||||||
if (ctx->is_ucs) {
|
|
||||||
/* QAT GEN4 uses single pass to treat AEAD as cipher
|
|
||||||
* operation
|
|
||||||
*/
|
|
||||||
struct icp_qat_fw_la_cipher_20_req_params *cipher_param_20 =
|
|
||||||
(void *)&req->serv_specif_rqpars;
|
|
||||||
cipher_param_20->spc_aad_addr = aad->iova;
|
|
||||||
cipher_param_20->spc_auth_res_addr = digest->iova;
|
|
||||||
} else {
|
|
||||||
cipher_param->spc_aad_addr = aad->iova;
|
|
||||||
cipher_param->spc_auth_res_addr = digest->iova;
|
|
||||||
}
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
switch (ctx->qat_hash_alg) {
|
|
||||||
case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
|
|
||||||
case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
|
|
||||||
ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
|
|
||||||
req->comn_hdr.serv_specif_flags,
|
|
||||||
ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
|
|
||||||
rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
|
|
||||||
ctx->cipher_iv.length);
|
|
||||||
aad_iova = aad->iova;
|
|
||||||
break;
|
|
||||||
case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
|
|
||||||
aad_data = aad->va;
|
|
||||||
aad_iova = aad->iova;
|
|
||||||
aad_ccm_real_len = 0;
|
|
||||||
aad_len_field_sz = 0;
|
|
||||||
msg_len_be = rte_bswap32((uint32_t)data_len -
|
|
||||||
ofs.ofs.cipher.head);
|
|
||||||
|
|
||||||
if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
|
|
||||||
aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
|
|
||||||
aad_ccm_real_len = ctx->aad_len -
|
|
||||||
ICP_QAT_HW_CCM_AAD_B0_LEN -
|
|
||||||
ICP_QAT_HW_CCM_AAD_LEN_INFO;
|
|
||||||
} else {
|
|
||||||
aad_data = iv->va;
|
|
||||||
aad_iova = iv->iova;
|
|
||||||
}
|
|
||||||
|
|
||||||
q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
|
|
||||||
aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
|
|
||||||
aad_len_field_sz, ctx->digest_length, q);
|
|
||||||
if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
|
|
||||||
memcpy(aad_data + ctx->cipher_iv.length +
|
|
||||||
ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
|
|
||||||
ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
|
|
||||||
(uint8_t *)&msg_len_be,
|
|
||||||
ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
|
|
||||||
} else {
|
|
||||||
memcpy(aad_data + ctx->cipher_iv.length +
|
|
||||||
ICP_QAT_HW_CCM_NONCE_OFFSET,
|
|
||||||
(uint8_t *)&msg_len_be +
|
|
||||||
(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
|
|
||||||
- q), q);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (aad_len_field_sz > 0) {
|
|
||||||
*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
|
|
||||||
rte_bswap16(aad_ccm_real_len);
|
|
||||||
|
|
||||||
if ((aad_ccm_real_len + aad_len_field_sz)
|
|
||||||
% ICP_QAT_HW_CCM_AAD_B0_LEN) {
|
|
||||||
uint8_t pad_len = 0;
|
|
||||||
uint8_t pad_idx = 0;
|
|
||||||
|
|
||||||
pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
|
|
||||||
((aad_ccm_real_len +
|
|
||||||
aad_len_field_sz) %
|
|
||||||
ICP_QAT_HW_CCM_AAD_B0_LEN);
|
|
||||||
pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
|
|
||||||
aad_ccm_real_len +
|
|
||||||
aad_len_field_sz;
|
|
||||||
memset(&aad_data[pad_idx], 0, pad_len);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
|
|
||||||
+ ICP_QAT_HW_CCM_NONCE_OFFSET,
|
|
||||||
(uint8_t *)iv->va +
|
|
||||||
ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
|
|
||||||
*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
|
|
||||||
q - ICP_QAT_HW_CCM_NONCE_OFFSET;
|
|
||||||
|
|
||||||
rte_memcpy((uint8_t *)aad->va +
|
|
||||||
ICP_QAT_HW_CCM_NONCE_OFFSET,
|
|
||||||
(uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
|
|
||||||
ctx->cipher_iv.length);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
cipher_param->cipher_offset = ofs.ofs.cipher.head;
|
|
||||||
cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
|
|
||||||
ofs.ofs.cipher.tail;
|
|
||||||
auth_param->auth_off = ofs.ofs.cipher.head;
|
|
||||||
auth_param->auth_len = cipher_param->cipher_length;
|
|
||||||
auth_param->auth_res_addr = digest->iova;
|
|
||||||
auth_param->u1.aad_adr = aad_iova;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline int
|
|
||||||
qat_sym_dp_enqueue_single_aead(void *qp_data, uint8_t *drv_ctx,
|
|
||||||
struct rte_crypto_vec *data, uint16_t n_data_vecs,
|
|
||||||
union rte_crypto_sym_ofs ofs,
|
|
||||||
struct rte_crypto_va_iova_ptr *iv,
|
|
||||||
struct rte_crypto_va_iova_ptr *digest,
|
|
||||||
struct rte_crypto_va_iova_ptr *aad,
|
|
||||||
void *user_data)
|
|
||||||
{
|
|
||||||
struct qat_qp *qp = qp_data;
|
|
||||||
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
|
||||||
struct qat_queue *tx_queue = &qp->tx_q;
|
|
||||||
struct qat_sym_session *ctx = dp_ctx->session;
|
|
||||||
struct icp_qat_fw_la_bulk_req *req;
|
|
||||||
int32_t data_len;
|
|
||||||
uint32_t tail = dp_ctx->tail;
|
|
||||||
|
|
||||||
req = (struct icp_qat_fw_la_bulk_req *)(
|
|
||||||
(uint8_t *)tx_queue->base_addr + tail);
|
|
||||||
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
|
||||||
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
|
||||||
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
|
|
||||||
data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
|
|
||||||
if (unlikely(data_len < 0))
|
|
||||||
return -1;
|
|
||||||
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
|
|
||||||
|
|
||||||
enqueue_one_aead_job(ctx, req, iv, digest, aad, ofs,
|
|
||||||
(uint32_t)data_len);
|
|
||||||
|
|
||||||
dp_ctx->tail = tail;
|
|
||||||
dp_ctx->cached_enqueue++;
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline uint32_t
|
|
||||||
qat_sym_dp_enqueue_aead_jobs(void *qp_data, uint8_t *drv_ctx,
|
|
||||||
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
|
|
||||||
void *user_data[], int *status)
|
|
||||||
{
|
|
||||||
struct qat_qp *qp = qp_data;
|
|
||||||
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
|
||||||
struct qat_queue *tx_queue = &qp->tx_q;
|
|
||||||
struct qat_sym_session *ctx = dp_ctx->session;
|
|
||||||
uint32_t i, n;
|
|
||||||
uint32_t tail;
|
|
||||||
struct icp_qat_fw_la_bulk_req *req;
|
|
||||||
int32_t data_len;
|
|
||||||
|
|
||||||
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
|
|
||||||
if (unlikely(n == 0)) {
|
|
||||||
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
|
|
||||||
*status = 0;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
tail = dp_ctx->tail;
|
|
||||||
|
|
||||||
for (i = 0; i < n; i++) {
|
|
||||||
req = (struct icp_qat_fw_la_bulk_req *)(
|
|
||||||
(uint8_t *)tx_queue->base_addr + tail);
|
|
||||||
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
|
|
||||||
|
|
||||||
data_len = qat_sym_dp_parse_data_vec(qp, req,
|
|
||||||
vec->src_sgl[i].vec,
|
|
||||||
vec->src_sgl[i].num);
|
|
||||||
if (unlikely(data_len < 0))
|
|
||||||
break;
|
|
||||||
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
|
|
||||||
enqueue_one_aead_job(ctx, req, &vec->iv[i], &vec->digest[i],
|
|
||||||
&vec->aad[i], ofs, (uint32_t)data_len);
|
|
||||||
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (unlikely(i < n))
|
|
||||||
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
|
|
||||||
|
|
||||||
dp_ctx->tail = tail;
|
|
||||||
dp_ctx->cached_enqueue += i;
|
|
||||||
*status = 0;
|
|
||||||
return i;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline uint32_t
|
|
||||||
qat_sym_dp_dequeue_burst(void *qp_data, uint8_t *drv_ctx,
|
|
||||||
rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
|
|
||||||
uint32_t max_nb_to_dequeue,
|
|
||||||
rte_cryptodev_raw_post_dequeue_t post_dequeue,
|
|
||||||
void **out_user_data, uint8_t is_user_data_array,
|
|
||||||
uint32_t *n_success_jobs, int *return_status)
|
|
||||||
{
|
|
||||||
struct qat_qp *qp = qp_data;
|
|
||||||
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
|
||||||
struct qat_queue *rx_queue = &qp->rx_q;
|
|
||||||
struct icp_qat_fw_comn_resp *resp;
|
|
||||||
void *resp_opaque;
|
|
||||||
uint32_t i, n, inflight;
|
|
||||||
uint32_t head;
|
|
||||||
uint8_t status;
|
|
||||||
|
|
||||||
*n_success_jobs = 0;
|
|
||||||
*return_status = 0;
|
|
||||||
head = dp_ctx->head;
|
|
||||||
|
|
||||||
inflight = qp->enqueued - qp->dequeued;
|
|
||||||
if (unlikely(inflight == 0))
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +
|
|
||||||
head);
|
|
||||||
/* no operation ready */
|
|
||||||
if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
resp_opaque = (void *)(uintptr_t)resp->opaque_data;
|
|
||||||
/* get the dequeue count */
|
|
||||||
if (get_dequeue_count) {
|
|
||||||
n = get_dequeue_count(resp_opaque);
|
|
||||||
if (unlikely(n == 0))
|
|
||||||
return 0;
|
|
||||||
} else {
|
|
||||||
if (unlikely(max_nb_to_dequeue == 0))
|
|
||||||
return 0;
|
|
||||||
n = max_nb_to_dequeue;
|
|
||||||
}
|
|
||||||
|
|
||||||
out_user_data[0] = resp_opaque;
|
|
||||||
status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
|
|
||||||
post_dequeue(resp_opaque, 0, status);
|
|
||||||
*n_success_jobs += status;
|
|
||||||
|
|
||||||
head = (head + rx_queue->msg_size) & rx_queue->modulo_mask;
|
|
||||||
|
|
||||||
/* we already finished dequeue when n == 1 */
|
|
||||||
if (unlikely(n == 1)) {
|
|
||||||
i = 1;
|
|
||||||
goto end_deq;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (is_user_data_array) {
|
|
||||||
for (i = 1; i < n; i++) {
|
|
||||||
resp = (struct icp_qat_fw_comn_resp *)(
|
|
||||||
(uint8_t *)rx_queue->base_addr + head);
|
|
||||||
if (unlikely(*(uint32_t *)resp ==
|
|
||||||
ADF_RING_EMPTY_SIG))
|
|
||||||
goto end_deq;
|
|
||||||
out_user_data[i] = (void *)(uintptr_t)resp->opaque_data;
|
|
||||||
status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
|
|
||||||
*n_success_jobs += status;
|
|
||||||
post_dequeue(out_user_data[i], i, status);
|
|
||||||
head = (head + rx_queue->msg_size) &
|
|
||||||
rx_queue->modulo_mask;
|
|
||||||
}
|
|
||||||
|
|
||||||
goto end_deq;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* opaque is not array */
|
|
||||||
for (i = 1; i < n; i++) {
|
|
||||||
resp = (struct icp_qat_fw_comn_resp *)(
|
|
||||||
(uint8_t *)rx_queue->base_addr + head);
|
|
||||||
status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
|
|
||||||
if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
|
|
||||||
goto end_deq;
|
|
||||||
head = (head + rx_queue->msg_size) &
|
|
||||||
rx_queue->modulo_mask;
|
|
||||||
post_dequeue(resp_opaque, i, status);
|
|
||||||
*n_success_jobs += status;
|
|
||||||
}
|
|
||||||
|
|
||||||
end_deq:
|
|
||||||
dp_ctx->head = head;
|
|
||||||
dp_ctx->cached_dequeue += i;
|
|
||||||
return i;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline void *
|
|
||||||
qat_sym_dp_dequeue(void *qp_data, uint8_t *drv_ctx, int *dequeue_status,
|
|
||||||
enum rte_crypto_op_status *op_status)
|
|
||||||
{
|
|
||||||
struct qat_qp *qp = qp_data;
|
|
||||||
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
|
||||||
struct qat_queue *rx_queue = &qp->rx_q;
|
|
||||||
register struct icp_qat_fw_comn_resp *resp;
|
|
||||||
|
|
||||||
resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +
|
|
||||||
dp_ctx->head);
|
|
||||||
|
|
||||||
if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
|
|
||||||
return NULL;
|
|
||||||
|
|
||||||
dp_ctx->head = (dp_ctx->head + rx_queue->msg_size) &
|
|
||||||
rx_queue->modulo_mask;
|
|
||||||
dp_ctx->cached_dequeue++;
|
|
||||||
|
|
||||||
*op_status = QAT_SYM_DP_IS_RESP_SUCCESS(resp) ?
|
|
||||||
RTE_CRYPTO_OP_STATUS_SUCCESS :
|
|
||||||
RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
|
|
||||||
*dequeue_status = 0;
|
|
||||||
return (void *)(uintptr_t)resp->opaque_data;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline int
|
|
||||||
qat_sym_dp_kick_tail(void *qp_data, uint8_t *drv_ctx, uint32_t n)
|
|
||||||
{
|
|
||||||
struct qat_qp *qp = qp_data;
|
|
||||||
struct qat_queue *tx_queue = &qp->tx_q;
|
|
||||||
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
|
||||||
|
|
||||||
if (unlikely(dp_ctx->cached_enqueue != n))
|
|
||||||
return -1;
|
|
||||||
|
|
||||||
qp->enqueued += n;
|
|
||||||
qp->stats.enqueued_count += n;
|
|
||||||
|
|
||||||
tx_queue->tail = dp_ctx->tail;
|
|
||||||
|
|
||||||
WRITE_CSR_RING_TAIL(qp->mmap_bar_addr,
|
|
||||||
tx_queue->hw_bundle_number,
|
|
||||||
tx_queue->hw_queue_number, tx_queue->tail);
|
|
||||||
tx_queue->csr_tail = tx_queue->tail;
|
|
||||||
dp_ctx->cached_enqueue = 0;
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static __rte_always_inline int
|
|
||||||
qat_sym_dp_update_head(void *qp_data, uint8_t *drv_ctx, uint32_t n)
|
|
||||||
{
|
|
||||||
struct qat_qp *qp = qp_data;
|
|
||||||
struct qat_queue *rx_queue = &qp->rx_q;
|
|
||||||
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
|
|
||||||
|
|
||||||
if (unlikely(dp_ctx->cached_dequeue != n))
|
|
||||||
return -1;
|
|
||||||
|
|
||||||
rx_queue->head = dp_ctx->head;
|
|
||||||
rx_queue->nb_processed_responses += n;
|
|
||||||
qp->dequeued += n;
|
|
||||||
qp->stats.dequeued_count += n;
|
|
||||||
if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) {
|
|
||||||
uint32_t old_head, new_head;
|
|
||||||
uint32_t max_head;
|
|
||||||
|
|
||||||
old_head = rx_queue->csr_head;
|
|
||||||
new_head = rx_queue->head;
|
|
||||||
max_head = qp->nb_descriptors * rx_queue->msg_size;
|
|
||||||
|
|
||||||
/* write out free descriptors */
|
|
||||||
void *cur_desc = (uint8_t *)rx_queue->base_addr + old_head;
|
|
||||||
|
|
||||||
if (new_head < old_head) {
|
|
||||||
memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE,
|
|
||||||
max_head - old_head);
|
|
||||||
memset(rx_queue->base_addr, ADF_RING_EMPTY_SIG_BYTE,
|
|
||||||
new_head);
|
|
||||||
} else {
|
|
||||||
memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head -
|
|
||||||
old_head);
|
|
||||||
}
|
|
||||||
rx_queue->nb_processed_responses = 0;
|
|
||||||
rx_queue->csr_head = new_head;
|
|
||||||
|
|
||||||
/* write current head to CSR */
|
|
||||||
WRITE_CSR_RING_HEAD(qp->mmap_bar_addr,
|
|
||||||
rx_queue->hw_bundle_number, rx_queue->hw_queue_number,
|
|
||||||
new_head);
|
|
||||||
}
|
|
||||||
|
|
||||||
dp_ctx->cached_dequeue = 0;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
int
|
|
||||||
qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
|
|
||||||
struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
|
|
||||||
enum rte_crypto_op_sess_type sess_type,
|
|
||||||
union rte_cryptodev_session_ctx session_ctx, uint8_t is_update)
|
|
||||||
{
|
|
||||||
struct qat_qp *qp;
|
|
||||||
struct qat_sym_session *ctx;
|
|
||||||
struct qat_sym_dp_ctx *dp_ctx;
|
|
||||||
|
|
||||||
qp = dev->data->queue_pairs[qp_id];
|
|
||||||
dp_ctx = (struct qat_sym_dp_ctx *)raw_dp_ctx->drv_ctx_data;
|
|
||||||
|
|
||||||
if (!is_update) {
|
|
||||||
memset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx) +
|
|
||||||
sizeof(struct qat_sym_dp_ctx));
|
|
||||||
raw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];
|
|
||||||
dp_ctx->tail = qp->tx_q.tail;
|
|
||||||
dp_ctx->head = qp->rx_q.head;
|
|
||||||
dp_ctx->cached_enqueue = dp_ctx->cached_dequeue = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (sess_type != RTE_CRYPTO_OP_WITH_SESSION)
|
|
||||||
return -EINVAL;
|
|
||||||
|
|
||||||
ctx = (struct qat_sym_session *)get_sym_session_private_data(
|
|
||||||
session_ctx.crypto_sess, qat_sym_driver_id);
|
|
||||||
|
|
||||||
dp_ctx->session = ctx;
|
|
||||||
|
|
||||||
raw_dp_ctx->enqueue_done = qat_sym_dp_kick_tail;
|
|
||||||
raw_dp_ctx->dequeue_burst = qat_sym_dp_dequeue_burst;
|
|
||||||
raw_dp_ctx->dequeue = qat_sym_dp_dequeue;
|
|
||||||
raw_dp_ctx->dequeue_done = qat_sym_dp_update_head;
|
|
||||||
|
|
||||||
if ((ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
|
|
||||||
ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) &&
|
|
||||||
!ctx->is_gmac) {
|
|
||||||
/* AES-GCM or AES-CCM */
|
|
||||||
if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
|
|
||||||
ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||
|
|
||||||
(ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128
|
|
||||||
&& ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE
|
|
||||||
&& ctx->qat_hash_alg ==
|
|
||||||
ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {
|
|
||||||
raw_dp_ctx->enqueue_burst =
|
|
||||||
qat_sym_dp_enqueue_aead_jobs;
|
|
||||||
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead;
|
|
||||||
} else {
|
|
||||||
raw_dp_ctx->enqueue_burst =
|
|
||||||
qat_sym_dp_enqueue_chain_jobs;
|
|
||||||
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_chain;
|
|
||||||
}
|
|
||||||
} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH || ctx->is_gmac) {
|
|
||||||
raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs;
|
|
||||||
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth;
|
|
||||||
} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
|
|
||||||
if (ctx->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE ||
|
|
||||||
ctx->qat_cipher_alg ==
|
|
||||||
ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305) {
|
|
||||||
raw_dp_ctx->enqueue_burst =
|
|
||||||
qat_sym_dp_enqueue_aead_jobs;
|
|
||||||
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead;
|
|
||||||
} else {
|
|
||||||
raw_dp_ctx->enqueue_burst =
|
|
||||||
qat_sym_dp_enqueue_cipher_jobs;
|
|
||||||
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_cipher;
|
|
||||||
}
|
|
||||||
} else
|
|
||||||
return -1;
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
int
|
|
||||||
qat_sym_get_dp_ctx_size(__rte_unused struct rte_cryptodev *dev)
|
|
||||||
{
|
|
||||||
return sizeof(struct qat_sym_dp_ctx);
|
|
||||||
}
|
|
||||||
Loading…
Reference in New Issue
Block a user