d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

crypto/qat: unify raw data path functions

Browse Source 
This patch unifies QAT's raw dp api implementations
to the same enqueue/dequeue methods used in crypto operations.
The specific functions for different QAT generation are updated
respectively. The qat_sym_hw_dp.c is removed as no longer required.

Signed-off-by: Kai Ji <kai.ji@intel.com>
Acked-by: Fan Zhang <roy.fan.zhang@intel.com>
This commit is contained in:
Kai Ji 2022-02-23 08:50:05 +08:00 committed by Akhil Goyal
parent fb3b9f4922
commit 85fec6fd96
10 changed files with 1137 additions and 994 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/common/qat/meson.build
View File

@ -74,7 +74,7 @@ endif
if qat_crypto
foreach f: ['qat_sym.c', 'qat_sym_session.c',
'qat_sym_hw_dp.c', 'qat_asym.c', 'qat_crypto.c',
'qat_asym.c', 'qat_crypto.c',
'dev/qat_sym_pmd_gen1.c',
'dev/qat_asym_pmd_gen1.c',
'dev/qat_crypto_pmd_gen2.c',

12
drivers/compress/qat/qat_comp_pmd.c
View File

@ -616,11 +616,18 @@ static struct rte_compressdev_ops compress_qat_dummy_ops = {
.private_xform_free = qat_comp_private_xform_free
};
static uint16_t
qat_comp_dequeue_burst(void *qp, struct rte_comp_op **ops, uint16_t nb_ops)
{
return qat_dequeue_op_burst(qp, (void **)ops, qat_comp_process_response,
nb_ops);
}
static uint16_t
qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
uint16_t nb_ops)
{
uint16_t ret = qat_dequeue_op_burst(qp, (void **)ops, NULL, nb_ops);
uint16_t ret = qat_comp_dequeue_burst(qp, ops, nb_ops);
struct qat_qp *tmp_qp = (struct qat_qp *)qp;
if (ret) {
@ -638,8 +645,7 @@ qat_comp_pmd_dequeue_first_op_burst(void *qp, struct rte_comp_op **ops,
} else {
tmp_qp->qat_dev->comp_dev->compressdev->dequeue_burst =
(compressdev_dequeue_pkt_burst_t)
qat_comp_pmd_enq_deq_dummy_op_burst;
qat_comp_dequeue_burst;
}
}
return ret;

2
drivers/crypto/qat/dev/qat_crypto_pmd_gen2.c
View File

@ -291,6 +291,8 @@ RTE_INIT(qat_sym_crypto_gen2_init)
qat_sym_crypto_cap_get_gen2;
qat_sym_gen_dev_ops[QAT_GEN2].set_session =
qat_sym_crypto_set_session_gen2;
qat_sym_gen_dev_ops[QAT_GEN2].set_raw_dp_ctx =
qat_sym_configure_raw_dp_ctx_gen1;
qat_sym_gen_dev_ops[QAT_GEN2].get_feature_flags =
qat_sym_crypto_feature_flags_get_gen1;

214
drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c
View File

@ -394,6 +394,218 @@ qat_sym_crypto_set_session_gen3(void *cdev __rte_unused, void *session)
return ret;
}
static int
qat_sym_dp_enqueue_single_aead_gen3(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *iv,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *aad,
void *user_data)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_op_cookie *cookie;
struct qat_sym_session *ctx = dp_ctx->session;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
uint32_t tail = dp_ctx->tail;
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
cookie = qp->op_cookies[tail >> tx_queue->trailz];
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
data, n_data_vecs, NULL, 0);
if (unlikely(data_len < 0))
return -1;
enqueue_one_aead_job_gen3(ctx, req, iv, digest, aad, ofs,
(uint32_t)data_len);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue++;
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv,
NULL, aad, digest);
#endif
return 0;
}
static uint32_t
qat_sym_dp_enqueue_aead_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
uint32_t i, n;
uint32_t tail;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
if (unlikely(n == 0)) {
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
*status = 0;
return 0;
}
tail = dp_ctx->tail;
for (i = 0; i < n; i++) {
struct qat_sym_op_cookie *cookie =
qp->op_cookies[tail >> tx_queue->trailz];
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
if (unlikely(data_len < 0))
break;
enqueue_one_aead_job_gen3(ctx, req, &vec->iv[i],
&vec->digest[i], &vec->aad[i], ofs,
(uint32_t)data_len);
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
vec->src_sgl[i].num, &vec->iv[i], NULL,
&vec->aad[i], &vec->digest[i]);
#endif
}
if (unlikely(i < n))
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue += i;
*status = 0;
return i;
}
static int
qat_sym_dp_enqueue_single_auth_gen3(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *iv __rte_unused,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *auth_iv,
void *user_data)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_op_cookie *cookie;
struct qat_sym_session *ctx = dp_ctx->session;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
uint32_t tail = dp_ctx->tail;
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
cookie = qp->op_cookies[tail >> tx_queue->trailz];
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
data, n_data_vecs, NULL, 0);
if (unlikely(data_len < 0))
return -1;
enqueue_one_auth_job_gen3(ctx, cookie, req, digest, auth_iv, ofs,
(uint32_t)data_len);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue++;
return 0;
}
static uint32_t
qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
uint32_t i, n;
uint32_t tail;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
if (unlikely(n == 0)) {
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
*status = 0;
return 0;
}
tail = dp_ctx->tail;
for (i = 0; i < n; i++) {
struct qat_sym_op_cookie *cookie =
qp->op_cookies[tail >> tx_queue->trailz];
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
if (unlikely(data_len < 0))
break;
enqueue_one_auth_job_gen3(ctx, cookie, req, &vec->digest[i],
&vec->auth_iv[i], ofs, (uint32_t)data_len);
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
}
if (unlikely(i < n))
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue += i;
*status = 0;
return i;
}
static int
qat_sym_configure_raw_dp_ctx_gen3(void *_raw_dp_ctx, void *_ctx)
{
struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;
struct qat_sym_session *ctx = _ctx;
int ret;
ret = qat_sym_configure_raw_dp_ctx_gen1(_raw_dp_ctx, _ctx);
if (ret < 0)
return ret;
if (ctx->is_single_pass) {
raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_aead_jobs_gen3;
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead_gen3;
} else if (ctx->is_single_pass_gmac) {
raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs_gen3;
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth_gen3;
}
return 0;
}
RTE_INIT(qat_sym_crypto_gen3_init)
{
qat_sym_gen_dev_ops[QAT_GEN3].cryptodev_ops = &qat_sym_crypto_ops_gen1;
@ -403,6 +615,8 @@ RTE_INIT(qat_sym_crypto_gen3_init)
qat_sym_crypto_feature_flags_get_gen1;
qat_sym_gen_dev_ops[QAT_GEN3].set_session =
qat_sym_crypto_set_session_gen3;
qat_sym_gen_dev_ops[QAT_GEN3].set_raw_dp_ctx =
qat_sym_configure_raw_dp_ctx_gen3;
#ifdef RTE_LIB_SECURITY
qat_sym_gen_dev_ops[QAT_GEN3].create_security_ctx =
qat_sym_create_security_gen1;

122
drivers/crypto/qat/dev/qat_crypto_pmd_gen4.c
View File

@ -223,6 +223,126 @@ qat_sym_crypto_set_session_gen4(void *cdev, void *session)
return ret;
}
static int
qat_sym_dp_enqueue_single_aead_gen4(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *iv,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *aad,
void *user_data)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_op_cookie *cookie;
struct qat_sym_session *ctx = dp_ctx->session;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
uint32_t tail = dp_ctx->tail;
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
cookie = qp->op_cookies[tail >> tx_queue->trailz];
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
data, n_data_vecs, NULL, 0);
if (unlikely(data_len < 0))
return -1;
enqueue_one_aead_job_gen4(ctx, req, iv, digest, aad, ofs,
(uint32_t)data_len);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue++;
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv,
NULL, aad, digest);
#endif
return 0;
}
static uint32_t
qat_sym_dp_enqueue_aead_jobs_gen4(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
uint32_t i, n;
uint32_t tail;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
if (unlikely(n == 0)) {
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
*status = 0;
return 0;
}
tail = dp_ctx->tail;
for (i = 0; i < n; i++) {
struct qat_sym_op_cookie *cookie =
qp->op_cookies[tail >> tx_queue->trailz];
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
if (unlikely(data_len < 0))
break;
enqueue_one_aead_job_gen4(ctx, req, &vec->iv[i],
&vec->digest[i], &vec->aad[i], ofs,
(uint32_t)data_len);
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
vec->src_sgl[i].num, &vec->iv[i], NULL,
&vec->aad[i], &vec->digest[i]);
#endif
}
if (unlikely(i < n))
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue += i;
*status = 0;
return i;
}
static int
qat_sym_configure_raw_dp_ctx_gen4(void *_raw_dp_ctx, void *_ctx)
{
struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;
struct qat_sym_session *ctx = _ctx;
int ret;
ret = qat_sym_configure_raw_dp_ctx_gen1(_raw_dp_ctx, _ctx);
if (ret < 0)
return ret;
if (ctx->is_single_pass && ctx->is_ucs) {
raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_aead_jobs_gen4;
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead_gen4;
}
return 0;
}
RTE_INIT(qat_sym_crypto_gen4_init)
{
qat_sym_gen_dev_ops[QAT_GEN4].cryptodev_ops = &qat_sym_crypto_ops_gen1;
@ -230,6 +350,8 @@ RTE_INIT(qat_sym_crypto_gen4_init)
qat_sym_crypto_cap_get_gen4;
qat_sym_gen_dev_ops[QAT_GEN4].set_session =
qat_sym_crypto_set_session_gen4;
qat_sym_gen_dev_ops[QAT_GEN4].set_raw_dp_ctx =
qat_sym_configure_raw_dp_ctx_gen4;
qat_sym_gen_dev_ops[QAT_GEN4].get_feature_flags =
qat_sym_crypto_feature_flags_get_gen1;
#ifdef RTE_LIB_SECURITY

78
drivers/crypto/qat/dev/qat_crypto_pmd_gens.h
View File

@ -839,6 +839,84 @@ int
qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
uint8_t *out_msg, void *op_cookie);
/* -----------------GEN 1 sym crypto raw data path APIs ---------------- */
int
qat_sym_dp_enqueue_single_cipher_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *iv,
struct rte_crypto_va_iova_ptr *digest __rte_unused,
struct rte_crypto_va_iova_ptr *aad __rte_unused,
void *user_data);
uint32_t
qat_sym_dp_enqueue_cipher_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status);
int
qat_sym_dp_enqueue_single_auth_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *iv __rte_unused,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *auth_iv,
void *user_data);
uint32_t
qat_sym_dp_enqueue_auth_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status);
int
qat_sym_dp_enqueue_single_chain_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *cipher_iv,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *auth_iv,
void *user_data);
uint32_t
qat_sym_dp_enqueue_chain_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status);
int
qat_sym_dp_enqueue_single_aead_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *iv,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *aad,
void *user_data);
uint32_t
qat_sym_dp_enqueue_aead_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status);
void *
qat_sym_dp_dequeue_single_gen1(void *qp_data, uint8_t *drv_ctx,
int *dequeue_status, enum rte_crypto_op_status *op_status);
uint32_t
qat_sym_dp_dequeue_burst_gen1(void *qp_data, uint8_t *drv_ctx,
rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
uint32_t max_nb_to_dequeue,
rte_cryptodev_raw_post_dequeue_t post_dequeue,
void **out_user_data, uint8_t is_user_data_array,
uint32_t *n_success_jobs, int *return_status);
int
qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
int
qat_sym_dp_dequeue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
int
qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx);
/* -----------------GENx control path APIs ---------------- */
uint64_t
qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);

656
drivers/crypto/qat/dev/qat_sym_pmd_gen1.c
View File

@ -146,6 +146,10 @@ struct rte_cryptodev_ops qat_sym_crypto_ops_gen1 = {
.sym_session_get_size = qat_sym_session_get_private_size,
.sym_session_configure = qat_sym_session_configure,
.sym_session_clear = qat_sym_session_clear,
/* Raw data-path API related operations */
.sym_get_raw_dp_ctx_size = qat_sym_get_dp_ctx_size,
.sym_configure_raw_dp_ctx = qat_sym_configure_dp_ctx,
};
static struct qat_capabilities_info
@ -448,6 +452,656 @@ qat_sym_create_security_gen1(void *cryptodev)
}
#endif
int
qat_sym_dp_enqueue_single_cipher_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *iv,
struct rte_crypto_va_iova_ptr *digest __rte_unused,
struct rte_crypto_va_iova_ptr *aad __rte_unused,
void *user_data)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
struct qat_sym_op_cookie *cookie;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
uint32_t tail = dp_ctx->tail;
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
cookie = qp->op_cookies[tail >> tx_queue->trailz];
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
data, n_data_vecs, NULL, 0);
if (unlikely(data_len < 0))
return -1;
enqueue_one_cipher_job_gen1(ctx, req, iv, ofs, (uint32_t)data_len);
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv,
NULL, NULL, NULL);
#endif
dp_ctx->tail = tail;
dp_ctx->cached_enqueue++;
return 0;
}
uint32_t
qat_sym_dp_enqueue_cipher_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
uint32_t i, n;
uint32_t tail;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
if (unlikely(n == 0)) {
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
*status = 0;
return 0;
}
tail = dp_ctx->tail;
for (i = 0; i < n; i++) {
struct qat_sym_op_cookie *cookie =
qp->op_cookies[tail >> tx_queue->trailz];
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
data_len = qat_sym_build_req_set_data(req, user_data[i],
cookie, vec->src_sgl[i].vec,
vec->src_sgl[i].num, NULL, 0);
if (unlikely(data_len < 0))
break;
enqueue_one_cipher_job_gen1(ctx, req, &vec->iv[i], ofs,
(uint32_t)data_len);
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
vec->src_sgl[i].num, &vec->iv[i],
NULL, NULL, NULL);
#endif
}
if (unlikely(i < n))
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue += i;
*status = 0;
return i;
}
int
qat_sym_dp_enqueue_single_auth_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *iv __rte_unused,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *auth_iv,
void *user_data)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_op_cookie *cookie;
struct qat_sym_session *ctx = dp_ctx->session;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
uint32_t tail = dp_ctx->tail;
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
cookie = qp->op_cookies[tail >> tx_queue->trailz];
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
data, n_data_vecs, NULL, 0);
if (unlikely(data_len < 0))
return -1;
enqueue_one_auth_job_gen1(ctx, req, digest, auth_iv, ofs,
(uint32_t)data_len);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue++;
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, NULL,
auth_iv, NULL, digest);
#endif
return 0;
}
uint32_t
qat_sym_dp_enqueue_auth_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
uint32_t i, n;
uint32_t tail;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
if (unlikely(n == 0)) {
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
*status = 0;
return 0;
}
tail = dp_ctx->tail;
for (i = 0; i < n; i++) {
struct qat_sym_op_cookie *cookie =
qp->op_cookies[tail >> tx_queue->trailz];
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
if (unlikely(data_len < 0))
break;
enqueue_one_auth_job_gen1(ctx, req, &vec->digest[i],
&vec->auth_iv[i], ofs, (uint32_t)data_len);
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
vec->src_sgl[i].num, NULL, &vec->auth_iv[i],
NULL, &vec->digest[i]);
#endif
}
if (unlikely(i < n))
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue += i;
*status = 0;
return i;
}
int
qat_sym_dp_enqueue_single_chain_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *cipher_iv,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *auth_iv,
void *user_data)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_op_cookie *cookie;
struct qat_sym_session *ctx = dp_ctx->session;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
uint32_t tail = dp_ctx->tail;
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
cookie = qp->op_cookies[tail >> tx_queue->trailz];
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
data, n_data_vecs, NULL, 0);
if (unlikely(data_len < 0))
return -1;
if (unlikely(enqueue_one_chain_job_gen1(ctx, req, data, n_data_vecs,
NULL, 0, cipher_iv, digest, auth_iv, ofs,
(uint32_t)data_len)))
return -1;
dp_ctx->tail = tail;
dp_ctx->cached_enqueue++;
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, cipher_iv,
auth_iv, NULL, digest);
#endif
return 0;
}
uint32_t
qat_sym_dp_enqueue_chain_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
uint32_t i, n;
uint32_t tail;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
if (unlikely(n == 0)) {
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
*status = 0;
return 0;
}
tail = dp_ctx->tail;
for (i = 0; i < n; i++) {
struct qat_sym_op_cookie *cookie =
qp->op_cookies[tail >> tx_queue->trailz];
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
if (unlikely(data_len < 0))
break;
if (unlikely(enqueue_one_chain_job_gen1(ctx, req,
vec->src_sgl[i].vec, vec->src_sgl[i].num,
NULL, 0,
&vec->iv[i], &vec->digest[i],
&vec->auth_iv[i], ofs, (uint32_t)data_len)))
break;
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
vec->src_sgl[i].num, &vec->iv[i],
&vec->auth_iv[i],
NULL, &vec->digest[i]);
#endif
}
if (unlikely(i < n))
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue += i;
*status = 0;
return i;
}
int
qat_sym_dp_enqueue_single_aead_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *iv,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *aad,
void *user_data)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_op_cookie *cookie;
struct qat_sym_session *ctx = dp_ctx->session;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
uint32_t tail = dp_ctx->tail;
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
cookie = qp->op_cookies[tail >> tx_queue->trailz];
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
data_len = qat_sym_build_req_set_data(req, user_data, cookie,
data, n_data_vecs, NULL, 0);
if (unlikely(data_len < 0))
return -1;
enqueue_one_aead_job_gen1(ctx, req, iv, digest, aad, ofs,
(uint32_t)data_len);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue++;
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv,
NULL, aad, digest);
#endif
return 0;
}
uint32_t
qat_sym_dp_enqueue_aead_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
uint32_t i, n;
uint32_t tail;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
if (unlikely(n == 0)) {
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
*status = 0;
return 0;
}
tail = dp_ctx->tail;
for (i = 0; i < n; i++) {
struct qat_sym_op_cookie *cookie =
qp->op_cookies[tail >> tx_queue->trailz];
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
data_len = qat_sym_build_req_set_data(req, user_data[i], cookie,
vec->src_sgl[i].vec, vec->src_sgl[i].num, NULL, 0);
if (unlikely(data_len < 0))
break;
enqueue_one_aead_job_gen1(ctx, req, &vec->iv[i],
&vec->digest[i], &vec->aad[i], ofs,
(uint32_t)data_len);
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
#if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
vec->src_sgl[i].num, &vec->iv[i], NULL,
&vec->aad[i], &vec->digest[i]);
#endif
}
if (unlikely(i < n))
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue += i;
*status = 0;
return i;
}
uint32_t
qat_sym_dp_dequeue_burst_gen1(void *qp_data, uint8_t *drv_ctx,
rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
uint32_t max_nb_to_dequeue,
rte_cryptodev_raw_post_dequeue_t post_dequeue,
void **out_user_data, uint8_t is_user_data_array,
uint32_t *n_success_jobs, int *return_status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *rx_queue = &qp->rx_q;
struct icp_qat_fw_comn_resp *resp;
void *resp_opaque;
uint32_t i, n, inflight;
uint32_t head;
uint8_t status;
*n_success_jobs = 0;
*return_status = 0;
head = dp_ctx->head;
inflight = qp->enqueued - qp->dequeued;
if (unlikely(inflight == 0))
return 0;
resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +
head);
/* no operation ready */
if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
return 0;
resp_opaque = (void *)(uintptr_t)resp->opaque_data;
/* get the dequeue count */
if (get_dequeue_count) {
n = get_dequeue_count(resp_opaque);
if (unlikely(n == 0))
return 0;
} else {
if (unlikely(max_nb_to_dequeue == 0))
return 0;
n = max_nb_to_dequeue;
}
out_user_data[0] = resp_opaque;
status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
post_dequeue(resp_opaque, 0, status);
*n_success_jobs += status;
head = (head + rx_queue->msg_size) & rx_queue->modulo_mask;
/* we already finished dequeue when n == 1 */
if (unlikely(n == 1)) {
i = 1;
goto end_deq;
}
if (is_user_data_array) {
for (i = 1; i < n; i++) {
resp = (struct icp_qat_fw_comn_resp *)(
(uint8_t *)rx_queue->base_addr + head);
if (unlikely(*(uint32_t *)resp ==
ADF_RING_EMPTY_SIG))
goto end_deq;
out_user_data[i] = (void *)(uintptr_t)resp->opaque_data;
status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
*n_success_jobs += status;
post_dequeue(out_user_data[i], i, status);
head = (head + rx_queue->msg_size) &
rx_queue->modulo_mask;
}
goto end_deq;
}
/* opaque is not array */
for (i = 1; i < n; i++) {
resp = (struct icp_qat_fw_comn_resp *)(
(uint8_t *)rx_queue->base_addr + head);
status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
goto end_deq;
head = (head + rx_queue->msg_size) &
rx_queue->modulo_mask;
post_dequeue(resp_opaque, i, status);
*n_success_jobs += status;
}
end_deq:
dp_ctx->head = head;
dp_ctx->cached_dequeue += i;
return i;
}
void *
qat_sym_dp_dequeue_single_gen1(void *qp_data, uint8_t *drv_ctx,
int *dequeue_status, enum rte_crypto_op_status *op_status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *rx_queue = &qp->rx_q;
register struct icp_qat_fw_comn_resp *resp;
resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +
dp_ctx->head);
if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
return NULL;
dp_ctx->head = (dp_ctx->head + rx_queue->msg_size) &
rx_queue->modulo_mask;
dp_ctx->cached_dequeue++;
*op_status = QAT_SYM_DP_IS_RESP_SUCCESS(resp) ?
RTE_CRYPTO_OP_STATUS_SUCCESS :
RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
*dequeue_status = 0;
return (void *)(uintptr_t)resp->opaque_data;
}
int
qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n)
{
struct qat_qp *qp = qp_data;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
if (unlikely(dp_ctx->cached_enqueue != n))
return -1;
qp->enqueued += n;
qp->stats.enqueued_count += n;
tx_queue->tail = dp_ctx->tail;
WRITE_CSR_RING_TAIL(qp->mmap_bar_addr,
tx_queue->hw_bundle_number,
tx_queue->hw_queue_number, tx_queue->tail);
tx_queue->csr_tail = tx_queue->tail;
dp_ctx->cached_enqueue = 0;
return 0;
}
int
qat_sym_dp_dequeue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n)
{
struct qat_qp *qp = qp_data;
struct qat_queue *rx_queue = &qp->rx_q;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
if (unlikely(dp_ctx->cached_dequeue != n))
return -1;
rx_queue->head = dp_ctx->head;
rx_queue->nb_processed_responses += n;
qp->dequeued += n;
qp->stats.dequeued_count += n;
if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) {
uint32_t old_head, new_head;
uint32_t max_head;
old_head = rx_queue->csr_head;
new_head = rx_queue->head;
max_head = qp->nb_descriptors * rx_queue->msg_size;
/* write out free descriptors */
void *cur_desc = (uint8_t *)rx_queue->base_addr + old_head;
if (new_head < old_head) {
memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE,
max_head - old_head);
memset(rx_queue->base_addr, ADF_RING_EMPTY_SIG_BYTE,
new_head);
} else {
memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head -
old_head);
}
rx_queue->nb_processed_responses = 0;
rx_queue->csr_head = new_head;
/* write current head to CSR */
WRITE_CSR_RING_HEAD(qp->mmap_bar_addr,
rx_queue->hw_bundle_number, rx_queue->hw_queue_number,
new_head);
}
dp_ctx->cached_dequeue = 0;
return 0;
}
int
qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx)
{
struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;
struct qat_sym_session *ctx = _ctx;
raw_dp_ctx->enqueue_done = qat_sym_dp_enqueue_done_gen1;
raw_dp_ctx->dequeue_burst = qat_sym_dp_dequeue_burst_gen1;
raw_dp_ctx->dequeue = qat_sym_dp_dequeue_single_gen1;
raw_dp_ctx->dequeue_done = qat_sym_dp_dequeue_done_gen1;
if ((ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) &&
!ctx->is_gmac) {
/* AES-GCM or AES-CCM */
if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||
(ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128
&& ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE
&& ctx->qat_hash_alg ==
ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {
raw_dp_ctx->enqueue_burst =
qat_sym_dp_enqueue_aead_jobs_gen1;
raw_dp_ctx->enqueue =
qat_sym_dp_enqueue_single_aead_gen1;
} else {
raw_dp_ctx->enqueue_burst =
qat_sym_dp_enqueue_chain_jobs_gen1;
raw_dp_ctx->enqueue =
qat_sym_dp_enqueue_single_chain_gen1;
}
} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH || ctx->is_gmac) {
raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs_gen1;
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth_gen1;
} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
if (ctx->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE ||
ctx->qat_cipher_alg ==
ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305) {
raw_dp_ctx->enqueue_burst =
qat_sym_dp_enqueue_aead_jobs_gen1;
raw_dp_ctx->enqueue =
qat_sym_dp_enqueue_single_aead_gen1;
} else {
raw_dp_ctx->enqueue_burst =
qat_sym_dp_enqueue_cipher_jobs_gen1;
raw_dp_ctx->enqueue =
qat_sym_dp_enqueue_single_cipher_gen1;
}
} else
return -1;
return 0;
}
int
qat_sym_crypto_set_session_gen1(void *cryptodev __rte_unused, void *session)
{
@ -518,6 +1172,8 @@ RTE_INIT(qat_sym_crypto_gen1_init)
qat_sym_crypto_cap_get_gen1;
qat_sym_gen_dev_ops[QAT_GEN1].set_session =
qat_sym_crypto_set_session_gen1;
qat_sym_gen_dev_ops[QAT_GEN1].set_raw_dp_ctx =
qat_sym_configure_raw_dp_ctx_gen1;
qat_sym_gen_dev_ops[QAT_GEN1].get_feature_flags =
qat_sym_crypto_feature_flags_get_gen1;
#ifdef RTE_LIB_SECURITY

3
drivers/crypto/qat/qat_crypto.h
View File

@ -53,11 +53,14 @@ typedef void * (*create_security_ctx_t)(void *cryptodev);
typedef int (*set_session_t)(void *cryptodev, void *session);
typedef int (*set_raw_dp_ctx_t)(void *raw_dp_ctx, void *ctx);
struct qat_crypto_gen_dev_ops {
get_feature_flags_t get_feature_flags;
get_capabilities_info_t get_capabilities;
struct rte_cryptodev_ops *cryptodev_ops;
set_session_t set_session;
set_raw_dp_ctx_t set_raw_dp_ctx;
#ifdef RTE_LIB_SECURITY
create_security_ctx_t create_security_ctx;
#endif

56
drivers/crypto/qat/qat_sym.c
View File

@ -89,7 +89,7 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
(void *)cdev, (void *)sess);
if (ret < 0) {
op->status =
RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
return -EINVAL;
}
}
@ -144,7 +144,7 @@ qat_sym_build_request(void *in_op, uint8_t *out_msg,
(void *)cdev, (void *)sess);
if (ret < 0) {
op->status =
RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
return -EINVAL;
}
}
@ -292,8 +292,7 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
if (internals->capa_mz == NULL) {
QAT_LOG(DEBUG,
"Error allocating memzone for capabilities, "
"destroying PMD for %s",
name);
"destroying PMD for %s", name);
ret = -EFAULT;
goto error;
}
@ -355,6 +354,55 @@ qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev)
return 0;
}
int
qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
enum rte_crypto_op_sess_type sess_type,
union rte_cryptodev_session_ctx session_ctx, uint8_t is_update)
{
struct qat_cryptodev_private *internals = dev->data->dev_private;
enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen;
struct qat_crypto_gen_dev_ops *gen_dev_ops =
&qat_sym_gen_dev_ops[qat_dev_gen];
struct qat_qp *qp;
struct qat_sym_session *ctx;
struct qat_sym_dp_ctx *dp_ctx;
if (!gen_dev_ops->set_raw_dp_ctx) {
QAT_LOG(ERR, "Device GEN %u does not support raw data path",
qat_dev_gen);
return -ENOTSUP;
}
qp = dev->data->queue_pairs[qp_id];
dp_ctx = (struct qat_sym_dp_ctx *)raw_dp_ctx->drv_ctx_data;
if (!is_update) {
memset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx) +
sizeof(struct qat_sym_dp_ctx));
raw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];
dp_ctx->tail = qp->tx_q.tail;
dp_ctx->head = qp->rx_q.head;
dp_ctx->cached_enqueue = dp_ctx->cached_dequeue = 0;
}
if (sess_type != RTE_CRYPTO_OP_WITH_SESSION)
return -EINVAL;
ctx = (struct qat_sym_session *)get_sym_session_private_data(
session_ctx.crypto_sess, qat_sym_driver_id);
dp_ctx->session = ctx;
return gen_dev_ops->set_raw_dp_ctx(raw_dp_ctx, ctx);
}
int
qat_sym_get_dp_ctx_size(struct rte_cryptodev *dev __rte_unused)
{
return sizeof(struct qat_sym_dp_ctx);
}
static struct cryptodev_driver qat_crypto_drv;
RTE_PMD_REGISTER_CRYPTO_DRIVER(qat_crypto_drv,
cryptodev_qat_sym_driver,

986
drivers/crypto/qat/qat_sym_hw_dp.c
View File

@ -1,986 +0,0 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2022 Intel Corporation
*/
#include <cryptodev_pmd.h>
#include "adf_transport_access_macros.h"
#include "icp_qat_fw.h"
#include "icp_qat_fw_la.h"
#include "qat_sym.h"
#include "qat_sym_session.h"
#include "qat_qp.h"
static __rte_always_inline int32_t
qat_sym_dp_parse_data_vec(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req,
struct rte_crypto_vec *data, uint16_t n_data_vecs)
{
struct qat_queue *tx_queue;
struct qat_sym_op_cookie *cookie;
struct qat_sgl *list;
uint32_t i;
uint32_t total_len;
if (likely(n_data_vecs == 1)) {
req->comn_mid.src_data_addr = req->comn_mid.dest_data_addr =
data[0].iova;
req->comn_mid.src_length = req->comn_mid.dst_length =
data[0].len;
return data[0].len;
}
if (n_data_vecs == 0 || n_data_vecs > QAT_SYM_SGL_MAX_NUMBER)
return -1;
total_len = 0;
tx_queue = &qp->tx_q;
ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
QAT_COMN_PTR_TYPE_SGL);
cookie = qp->op_cookies[tx_queue->tail >> tx_queue->trailz];
list = (struct qat_sgl *)&cookie->qat_sgl_src;
for (i = 0; i < n_data_vecs; i++) {
list->buffers[i].len = data[i].len;
list->buffers[i].resrvd = 0;
list->buffers[i].addr = data[i].iova;
if (total_len + data[i].len > UINT32_MAX) {
QAT_DP_LOG(ERR, "Message too long");
return -1;
}
total_len += data[i].len;
}
list->num_bufs = i;
req->comn_mid.src_data_addr = req->comn_mid.dest_data_addr =
cookie->qat_sgl_src_phys_addr;
req->comn_mid.src_length = req->comn_mid.dst_length = 0;
return total_len;
}
static __rte_always_inline void
set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
struct icp_qat_fw_la_bulk_req *qat_req)
{
/* copy IV into request if it fits */
if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
iv_len);
else {
ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
qat_req->comn_hdr.serv_specif_flags,
ICP_QAT_FW_CIPH_IV_64BIT_PTR);
cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
}
}
#define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
(ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
static __rte_always_inline void
qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
{
uint32_t i;
for (i = 0; i < n; i++)
sta[i] = status;
}
#define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
static __rte_always_inline void
enqueue_one_cipher_job(struct qat_sym_session *ctx,
struct icp_qat_fw_la_bulk_req *req,
struct rte_crypto_va_iova_ptr *iv,
union rte_crypto_sym_ofs ofs, uint32_t data_len)
{
struct icp_qat_fw_la_cipher_req_params *cipher_param;
cipher_param = (void *)&req->serv_specif_rqpars;
/* cipher IV */
set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
cipher_param->cipher_offset = ofs.ofs.cipher.head;
cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
ofs.ofs.cipher.tail;
}
static __rte_always_inline int
qat_sym_dp_enqueue_single_cipher(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *iv,
struct rte_crypto_va_iova_ptr *digest __rte_unused,
struct rte_crypto_va_iova_ptr *aad __rte_unused,
void *user_data)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
uint32_t tail = dp_ctx->tail;
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
if (unlikely(data_len < 0))
return -1;
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
enqueue_one_cipher_job(ctx, req, iv, ofs, (uint32_t)data_len);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue++;
return 0;
}
static __rte_always_inline uint32_t
qat_sym_dp_enqueue_cipher_jobs(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
uint32_t i, n;
uint32_t tail;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
if (unlikely(n == 0)) {
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
*status = 0;
return 0;
}
tail = dp_ctx->tail;
for (i = 0; i < n; i++) {
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
data_len = qat_sym_dp_parse_data_vec(qp, req,
vec->src_sgl[i].vec,
vec->src_sgl[i].num);
if (unlikely(data_len < 0))
break;
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
enqueue_one_cipher_job(ctx, req, &vec->iv[i], ofs,
(uint32_t)data_len);
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
}
if (unlikely(i < n))
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue += i;
*status = 0;
return i;
}
static __rte_always_inline void
enqueue_one_auth_job(struct qat_sym_session *ctx,
struct icp_qat_fw_la_bulk_req *req,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *auth_iv,
union rte_crypto_sym_ofs ofs, uint32_t data_len)
{
struct icp_qat_fw_la_cipher_req_params *cipher_param;
struct icp_qat_fw_la_auth_req_params *auth_param;
cipher_param = (void *)&req->serv_specif_rqpars;
auth_param = (void *)((uint8_t *)cipher_param +
ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
auth_param->auth_off = ofs.ofs.auth.head;
auth_param->auth_len = data_len - ofs.ofs.auth.head -
ofs.ofs.auth.tail;
auth_param->auth_res_addr = digest->iova;
switch (ctx->qat_hash_alg) {
case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
auth_param->u1.aad_adr = auth_iv->iova;
break;
case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
req->comn_hdr.serv_specif_flags,
ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
ctx->auth_iv.length);
break;
default:
break;
}
}
static __rte_always_inline int
qat_sym_dp_enqueue_single_auth(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *iv __rte_unused,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *auth_iv,
void *user_data)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
uint32_t tail = dp_ctx->tail;
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
if (unlikely(data_len < 0))
return -1;
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
enqueue_one_auth_job(ctx, req, digest, auth_iv, ofs,
(uint32_t)data_len);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue++;
return 0;
}
static __rte_always_inline uint32_t
qat_sym_dp_enqueue_auth_jobs(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
uint32_t i, n;
uint32_t tail;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
if (unlikely(n == 0)) {
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
*status = 0;
return 0;
}
tail = dp_ctx->tail;
for (i = 0; i < n; i++) {
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
data_len = qat_sym_dp_parse_data_vec(qp, req,
vec->src_sgl[i].vec,
vec->src_sgl[i].num);
if (unlikely(data_len < 0))
break;
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
enqueue_one_auth_job(ctx, req, &vec->digest[i],
&vec->auth_iv[i], ofs, (uint32_t)data_len);
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
}
if (unlikely(i < n))
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue += i;
*status = 0;
return i;
}
static __rte_always_inline int
enqueue_one_chain_job(struct qat_sym_session *ctx,
struct icp_qat_fw_la_bulk_req *req,
struct rte_crypto_vec *data,
uint16_t n_data_vecs,
struct rte_crypto_va_iova_ptr *cipher_iv,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *auth_iv,
union rte_crypto_sym_ofs ofs, uint32_t data_len)
{
struct icp_qat_fw_la_cipher_req_params *cipher_param;
struct icp_qat_fw_la_auth_req_params *auth_param;
rte_iova_t auth_iova_end;
int32_t cipher_len, auth_len;
cipher_param = (void *)&req->serv_specif_rqpars;
auth_param = (void *)((uint8_t *)cipher_param +
ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
cipher_len = data_len - ofs.ofs.cipher.head -
ofs.ofs.cipher.tail;
auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
if (unlikely(cipher_len < 0 || auth_len < 0))
return -1;
cipher_param->cipher_offset = ofs.ofs.cipher.head;
cipher_param->cipher_length = cipher_len;
set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
auth_param->auth_off = ofs.ofs.auth.head;
auth_param->auth_len = auth_len;
auth_param->auth_res_addr = digest->iova;
switch (ctx->qat_hash_alg) {
case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
auth_param->u1.aad_adr = auth_iv->iova;
break;
case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
break;
default:
break;
}
if (unlikely(n_data_vecs > 1)) {
int auth_end_get = 0, i = n_data_vecs - 1;
struct rte_crypto_vec *cvec = &data[0];
uint32_t len;
len = data_len - ofs.ofs.auth.tail;
while (i >= 0 && len > 0) {
if (cvec->len >= len) {
auth_iova_end = cvec->iova + len;
len = 0;
auth_end_get = 1;
break;
}
len -= cvec->len;
i--;
cvec++;
}
if (unlikely(auth_end_get == 0))
return -1;
} else
auth_iova_end = data[0].iova + auth_param->auth_off +
auth_param->auth_len;
/* Then check if digest-encrypted conditions are met */
if ((auth_param->auth_off + auth_param->auth_len <
cipher_param->cipher_offset +
cipher_param->cipher_length) &&
(digest->iova == auth_iova_end)) {
/* Handle partial digest encryption */
if (cipher_param->cipher_offset +
cipher_param->cipher_length <
auth_param->auth_off +
auth_param->auth_len +
ctx->digest_length)
req->comn_mid.dst_length =
req->comn_mid.src_length =
auth_param->auth_off +
auth_param->auth_len +
ctx->digest_length;
struct icp_qat_fw_comn_req_hdr *header =
&req->comn_hdr;
ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
header->serv_specif_flags,
ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
}
return 0;
}
static __rte_always_inline int
qat_sym_dp_enqueue_single_chain(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *cipher_iv,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *auth_iv,
void *user_data)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
uint32_t tail = dp_ctx->tail;
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
if (unlikely(data_len < 0))
return -1;
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
if (unlikely(enqueue_one_chain_job(ctx, req, data, n_data_vecs,
cipher_iv, digest, auth_iv, ofs, (uint32_t)data_len)))
return -1;
dp_ctx->tail = tail;
dp_ctx->cached_enqueue++;
return 0;
}
static __rte_always_inline uint32_t
qat_sym_dp_enqueue_chain_jobs(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
uint32_t i, n;
uint32_t tail;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
if (unlikely(n == 0)) {
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
*status = 0;
return 0;
}
tail = dp_ctx->tail;
for (i = 0; i < n; i++) {
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
data_len = qat_sym_dp_parse_data_vec(qp, req,
vec->src_sgl[i].vec,
vec->src_sgl[i].num);
if (unlikely(data_len < 0))
break;
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
if (unlikely(enqueue_one_chain_job(ctx, req,
vec->src_sgl[i].vec, vec->src_sgl[i].num,
&vec->iv[i], &vec->digest[i],
&vec->auth_iv[i], ofs, (uint32_t)data_len)))
break;
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
}
if (unlikely(i < n))
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue += i;
*status = 0;
return i;
}
static __rte_always_inline void
enqueue_one_aead_job(struct qat_sym_session *ctx,
struct icp_qat_fw_la_bulk_req *req,
struct rte_crypto_va_iova_ptr *iv,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *aad,
union rte_crypto_sym_ofs ofs, uint32_t data_len)
{
struct icp_qat_fw_la_cipher_req_params *cipher_param =
(void *)&req->serv_specif_rqpars;
struct icp_qat_fw_la_auth_req_params *auth_param =
(void *)((uint8_t *)&req->serv_specif_rqpars +
ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
uint8_t *aad_data;
uint8_t aad_ccm_real_len;
uint8_t aad_len_field_sz;
uint32_t msg_len_be;
rte_iova_t aad_iova = 0;
uint8_t q;
/* CPM 1.7 uses single pass to treat AEAD as cipher operation */
if (ctx->is_single_pass) {
enqueue_one_cipher_job(ctx, req, iv, ofs, data_len);
if (ctx->is_ucs) {
/* QAT GEN4 uses single pass to treat AEAD as cipher
* operation
*/
struct icp_qat_fw_la_cipher_20_req_params *cipher_param_20 =
(void *)&req->serv_specif_rqpars;
cipher_param_20->spc_aad_addr = aad->iova;
cipher_param_20->spc_auth_res_addr = digest->iova;
} else {
cipher_param->spc_aad_addr = aad->iova;
cipher_param->spc_auth_res_addr = digest->iova;
}
return;
}
switch (ctx->qat_hash_alg) {
case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
req->comn_hdr.serv_specif_flags,
ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
ctx->cipher_iv.length);
aad_iova = aad->iova;
break;
case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
aad_data = aad->va;
aad_iova = aad->iova;
aad_ccm_real_len = 0;
aad_len_field_sz = 0;
msg_len_be = rte_bswap32((uint32_t)data_len -
ofs.ofs.cipher.head);
if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
aad_ccm_real_len = ctx->aad_len -
ICP_QAT_HW_CCM_AAD_B0_LEN -
ICP_QAT_HW_CCM_AAD_LEN_INFO;
} else {
aad_data = iv->va;
aad_iova = iv->iova;
}
q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
aad_len_field_sz, ctx->digest_length, q);
if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
memcpy(aad_data + ctx->cipher_iv.length +
ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
(uint8_t *)&msg_len_be,
ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
} else {
memcpy(aad_data + ctx->cipher_iv.length +
ICP_QAT_HW_CCM_NONCE_OFFSET,
(uint8_t *)&msg_len_be +
(ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
- q), q);
}
if (aad_len_field_sz > 0) {
*(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
rte_bswap16(aad_ccm_real_len);
if ((aad_ccm_real_len + aad_len_field_sz)
% ICP_QAT_HW_CCM_AAD_B0_LEN) {
uint8_t pad_len = 0;
uint8_t pad_idx = 0;
pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
((aad_ccm_real_len +
aad_len_field_sz) %
ICP_QAT_HW_CCM_AAD_B0_LEN);
pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
aad_ccm_real_len +
aad_len_field_sz;
memset(&aad_data[pad_idx], 0, pad_len);
}
}
rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
+ ICP_QAT_HW_CCM_NONCE_OFFSET,
(uint8_t *)iv->va +
ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
*(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
q - ICP_QAT_HW_CCM_NONCE_OFFSET;
rte_memcpy((uint8_t *)aad->va +
ICP_QAT_HW_CCM_NONCE_OFFSET,
(uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
ctx->cipher_iv.length);
break;
default:
break;
}
cipher_param->cipher_offset = ofs.ofs.cipher.head;
cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
ofs.ofs.cipher.tail;
auth_param->auth_off = ofs.ofs.cipher.head;
auth_param->auth_len = cipher_param->cipher_length;
auth_param->auth_res_addr = digest->iova;
auth_param->u1.aad_adr = aad_iova;
}
static __rte_always_inline int
qat_sym_dp_enqueue_single_aead(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_vec *data, uint16_t n_data_vecs,
union rte_crypto_sym_ofs ofs,
struct rte_crypto_va_iova_ptr *iv,
struct rte_crypto_va_iova_ptr *digest,
struct rte_crypto_va_iova_ptr *aad,
void *user_data)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
uint32_t tail = dp_ctx->tail;
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
if (unlikely(data_len < 0))
return -1;
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
enqueue_one_aead_job(ctx, req, iv, digest, aad, ofs,
(uint32_t)data_len);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue++;
return 0;
}
static __rte_always_inline uint32_t
qat_sym_dp_enqueue_aead_jobs(void *qp_data, uint8_t *drv_ctx,
struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
void *user_data[], int *status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_session *ctx = dp_ctx->session;
uint32_t i, n;
uint32_t tail;
struct icp_qat_fw_la_bulk_req *req;
int32_t data_len;
n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
if (unlikely(n == 0)) {
qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
*status = 0;
return 0;
}
tail = dp_ctx->tail;
for (i = 0; i < n; i++) {
req = (struct icp_qat_fw_la_bulk_req *)(
(uint8_t *)tx_queue->base_addr + tail);
rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
data_len = qat_sym_dp_parse_data_vec(qp, req,
vec->src_sgl[i].vec,
vec->src_sgl[i].num);
if (unlikely(data_len < 0))
break;
req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
enqueue_one_aead_job(ctx, req, &vec->iv[i], &vec->digest[i],
&vec->aad[i], ofs, (uint32_t)data_len);
tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
}
if (unlikely(i < n))
qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
dp_ctx->tail = tail;
dp_ctx->cached_enqueue += i;
*status = 0;
return i;
}
static __rte_always_inline uint32_t
qat_sym_dp_dequeue_burst(void *qp_data, uint8_t *drv_ctx,
rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
uint32_t max_nb_to_dequeue,
rte_cryptodev_raw_post_dequeue_t post_dequeue,
void **out_user_data, uint8_t is_user_data_array,
uint32_t *n_success_jobs, int *return_status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *rx_queue = &qp->rx_q;
struct icp_qat_fw_comn_resp *resp;
void *resp_opaque;
uint32_t i, n, inflight;
uint32_t head;
uint8_t status;
*n_success_jobs = 0;
*return_status = 0;
head = dp_ctx->head;
inflight = qp->enqueued - qp->dequeued;
if (unlikely(inflight == 0))
return 0;
resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +
head);
/* no operation ready */
if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
return 0;
resp_opaque = (void *)(uintptr_t)resp->opaque_data;
/* get the dequeue count */
if (get_dequeue_count) {
n = get_dequeue_count(resp_opaque);
if (unlikely(n == 0))
return 0;
} else {
if (unlikely(max_nb_to_dequeue == 0))
return 0;
n = max_nb_to_dequeue;
}
out_user_data[0] = resp_opaque;
status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
post_dequeue(resp_opaque, 0, status);
*n_success_jobs += status;
head = (head + rx_queue->msg_size) & rx_queue->modulo_mask;
/* we already finished dequeue when n == 1 */
if (unlikely(n == 1)) {
i = 1;
goto end_deq;
}
if (is_user_data_array) {
for (i = 1; i < n; i++) {
resp = (struct icp_qat_fw_comn_resp *)(
(uint8_t *)rx_queue->base_addr + head);
if (unlikely(*(uint32_t *)resp ==
ADF_RING_EMPTY_SIG))
goto end_deq;
out_user_data[i] = (void *)(uintptr_t)resp->opaque_data;
status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
*n_success_jobs += status;
post_dequeue(out_user_data[i], i, status);
head = (head + rx_queue->msg_size) &
rx_queue->modulo_mask;
}
goto end_deq;
}
/* opaque is not array */
for (i = 1; i < n; i++) {
resp = (struct icp_qat_fw_comn_resp *)(
(uint8_t *)rx_queue->base_addr + head);
status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
goto end_deq;
head = (head + rx_queue->msg_size) &
rx_queue->modulo_mask;
post_dequeue(resp_opaque, i, status);
*n_success_jobs += status;
}
end_deq:
dp_ctx->head = head;
dp_ctx->cached_dequeue += i;
return i;
}
static __rte_always_inline void *
qat_sym_dp_dequeue(void *qp_data, uint8_t *drv_ctx, int *dequeue_status,
enum rte_crypto_op_status *op_status)
{
struct qat_qp *qp = qp_data;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
struct qat_queue *rx_queue = &qp->rx_q;
register struct icp_qat_fw_comn_resp *resp;
resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +
dp_ctx->head);
if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
return NULL;
dp_ctx->head = (dp_ctx->head + rx_queue->msg_size) &
rx_queue->modulo_mask;
dp_ctx->cached_dequeue++;
*op_status = QAT_SYM_DP_IS_RESP_SUCCESS(resp) ?
RTE_CRYPTO_OP_STATUS_SUCCESS :
RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
*dequeue_status = 0;
return (void *)(uintptr_t)resp->opaque_data;
}
static __rte_always_inline int
qat_sym_dp_kick_tail(void *qp_data, uint8_t *drv_ctx, uint32_t n)
{
struct qat_qp *qp = qp_data;
struct qat_queue *tx_queue = &qp->tx_q;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
if (unlikely(dp_ctx->cached_enqueue != n))
return -1;
qp->enqueued += n;
qp->stats.enqueued_count += n;
tx_queue->tail = dp_ctx->tail;
WRITE_CSR_RING_TAIL(qp->mmap_bar_addr,
tx_queue->hw_bundle_number,
tx_queue->hw_queue_number, tx_queue->tail);
tx_queue->csr_tail = tx_queue->tail;
dp_ctx->cached_enqueue = 0;
return 0;
}
static __rte_always_inline int
qat_sym_dp_update_head(void *qp_data, uint8_t *drv_ctx, uint32_t n)
{
struct qat_qp *qp = qp_data;
struct qat_queue *rx_queue = &qp->rx_q;
struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
if (unlikely(dp_ctx->cached_dequeue != n))
return -1;
rx_queue->head = dp_ctx->head;
rx_queue->nb_processed_responses += n;
qp->dequeued += n;
qp->stats.dequeued_count += n;
if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) {
uint32_t old_head, new_head;
uint32_t max_head;
old_head = rx_queue->csr_head;
new_head = rx_queue->head;
max_head = qp->nb_descriptors * rx_queue->msg_size;
/* write out free descriptors */
void *cur_desc = (uint8_t *)rx_queue->base_addr + old_head;
if (new_head < old_head) {
memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE,
max_head - old_head);
memset(rx_queue->base_addr, ADF_RING_EMPTY_SIG_BYTE,
new_head);
} else {
memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head -
old_head);
}
rx_queue->nb_processed_responses = 0;
rx_queue->csr_head = new_head;
/* write current head to CSR */
WRITE_CSR_RING_HEAD(qp->mmap_bar_addr,
rx_queue->hw_bundle_number, rx_queue->hw_queue_number,
new_head);
}
dp_ctx->cached_dequeue = 0;
return 0;
}
int
qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
enum rte_crypto_op_sess_type sess_type,
union rte_cryptodev_session_ctx session_ctx, uint8_t is_update)
{
struct qat_qp *qp;
struct qat_sym_session *ctx;
struct qat_sym_dp_ctx *dp_ctx;
qp = dev->data->queue_pairs[qp_id];
dp_ctx = (struct qat_sym_dp_ctx *)raw_dp_ctx->drv_ctx_data;
if (!is_update) {
memset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx) +
sizeof(struct qat_sym_dp_ctx));
raw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];
dp_ctx->tail = qp->tx_q.tail;
dp_ctx->head = qp->rx_q.head;
dp_ctx->cached_enqueue = dp_ctx->cached_dequeue = 0;
}
if (sess_type != RTE_CRYPTO_OP_WITH_SESSION)
return -EINVAL;
ctx = (struct qat_sym_session *)get_sym_session_private_data(
session_ctx.crypto_sess, qat_sym_driver_id);
dp_ctx->session = ctx;
raw_dp_ctx->enqueue_done = qat_sym_dp_kick_tail;
raw_dp_ctx->dequeue_burst = qat_sym_dp_dequeue_burst;
raw_dp_ctx->dequeue = qat_sym_dp_dequeue;
raw_dp_ctx->dequeue_done = qat_sym_dp_update_head;
if ((ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) &&
!ctx->is_gmac) {
/* AES-GCM or AES-CCM */
if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||
(ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128
&& ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE
&& ctx->qat_hash_alg ==
ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {
raw_dp_ctx->enqueue_burst =
qat_sym_dp_enqueue_aead_jobs;
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead;
} else {
raw_dp_ctx->enqueue_burst =
qat_sym_dp_enqueue_chain_jobs;
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_chain;
}
} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH || ctx->is_gmac) {
raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs;
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth;
} else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
if (ctx->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE ||
ctx->qat_cipher_alg ==
ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305) {
raw_dp_ctx->enqueue_burst =
qat_sym_dp_enqueue_aead_jobs;
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead;
} else {
raw_dp_ctx->enqueue_burst =
qat_sym_dp_enqueue_cipher_jobs;
raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_cipher;
}
} else
return -1;
return 0;
}
int
qat_sym_get_dp_ctx_size(__rte_unused struct rte_cryptodev *dev)
{
return sizeof(struct qat_sym_dp_ctx);
}