d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

crypto/qat: unify device private data structure

Browse Source 
This patch unifies the QAT symmetric and asymmetric device
private data structures and functions.

Signed-off-by: Arek Kusztal <arkadiuszx.kusztal@intel.com>
Signed-off-by: Fan Zhang <roy.fan.zhang@intel.com>
Signed-off-by: Kai Ji <kai.ji@intel.com>
Acked-by: Ciara Power <ciara.power@intel.com>
This commit is contained in:
Fan Zhang 2021-11-04 10:34:55 +00:00 committed by Akhil Goyal
parent 2d148597ce
commit f0f369a685
11 changed files with 366 additions and 447 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
drivers/common/qat/meson.build
View File

@ -74,7 +74,8 @@ endif
if qat_crypto
foreach f: ['qat_sym_pmd.c', 'qat_sym.c', 'qat_sym_session.c',
'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c']
'qat_sym_hw_dp.c', 'qat_asym_pmd.c', 'qat_asym.c', 'qat_crypto.c',
]
sources += files(join_paths(qat_crypto_relpath, f))
endforeach
deps += ['security']

15
drivers/common/qat/qat_common.c
View File

@ -6,6 +6,21 @@
#include "qat_device.h"
#include "qat_logs.h"
const char *
qat_service_get_str(enum qat_service_type type)
{
switch (type) {
case QAT_SERVICE_SYMMETRIC:
return "sym";
case QAT_SERVICE_ASYMMETRIC:
return "asym";
case QAT_SERVICE_COMPRESSION:
return "comp";
default:
return "invalid";
}
}
int
qat_sgl_fill_array(struct rte_mbuf *buf, int64_t offset,
void *list_in, uint32_t data_len,

3
drivers/common/qat/qat_common.h
View File

@ -91,4 +91,7 @@ void
qat_stats_reset(struct qat_pci_device *dev,
enum qat_service_type service);
const char *
qat_service_get_str(enum qat_service_type type);
#endif /* _QAT_COMMON_H_ */

7
drivers/common/qat/qat_device.h
View File

@ -76,8 +76,7 @@ struct qat_device_info {
extern struct qat_device_info qat_pci_devs[];
struct qat_sym_dev_private;
struct qat_asym_dev_private;
struct qat_cryptodev_private;
struct qat_comp_dev_private;
/*
@ -106,14 +105,14 @@ struct qat_pci_device {
/**< links to qps set up for each service, index same as on API */
/* Data relating to symmetric crypto service */
struct qat_sym_dev_private *sym_dev;
struct qat_cryptodev_private *sym_dev;
/**< link back to cryptodev private data */
int qat_sym_driver_id;
/**< Symmetric driver id used by this device */
/* Data relating to asymmetric crypto service */
struct qat_asym_dev_private *asym_dev;
struct qat_cryptodev_private *asym_dev;
/**< link back to cryptodev private data */
int qat_asym_driver_id;

216
drivers/crypto/qat/qat_asym_pmd.c
View File

@ -6,6 +6,7 @@
#include "qat_logs.h"
#include "qat_crypto.h"
#include "qat_asym.h"
#include "qat_asym_pmd.h"
#include "qat_sym_capabilities.h"
@ -18,190 +19,45 @@ static const struct rte_cryptodev_capabilities qat_gen1_asym_capabilities[] = {
RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
};
static int qat_asym_qp_release(struct rte_cryptodev *dev,
uint16_t queue_pair_id);
static int qat_asym_dev_config(__rte_unused struct rte_cryptodev *dev,
__rte_unused struct rte_cryptodev_config *config)
void
qat_asym_init_op_cookie(void *op_cookie)
{
return 0;
}
int j;
struct qat_asym_op_cookie *cookie = op_cookie;
static int qat_asym_dev_start(__rte_unused struct rte_cryptodev *dev)
{
return 0;
}
cookie->input_addr = rte_mempool_virt2iova(cookie) +
offsetof(struct qat_asym_op_cookie,
input_params_ptrs);
static void qat_asym_dev_stop(__rte_unused struct rte_cryptodev *dev)
{
cookie->output_addr = rte_mempool_virt2iova(cookie) +
offsetof(struct qat_asym_op_cookie,
output_params_ptrs);
}
static int qat_asym_dev_close(struct rte_cryptodev *dev)
{
int i, ret;
for (i = 0; i < dev->data->nb_queue_pairs; i++) {
ret = qat_asym_qp_release(dev, i);
if (ret < 0)
return ret;
}
return 0;
}
static void qat_asym_dev_info_get(struct rte_cryptodev *dev,
struct rte_cryptodev_info *info)
{
struct qat_asym_dev_private *internals = dev->data->dev_private;
struct qat_pci_device *qat_dev = internals->qat_dev;
if (info != NULL) {
info->max_nb_queue_pairs = qat_qps_per_service(qat_dev,
QAT_SERVICE_ASYMMETRIC);
info->feature_flags = dev->feature_flags;
info->capabilities = internals->qat_dev_capabilities;
info->driver_id = qat_asym_driver_id;
/* No limit of number of sessions */
info->sym.max_nb_sessions = 0;
}
}
static void qat_asym_stats_get(struct rte_cryptodev *dev,
struct rte_cryptodev_stats *stats)
{
struct qat_common_stats qat_stats = {0};
struct qat_asym_dev_private *qat_priv;
if (stats == NULL || dev == NULL) {
QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev);
return;
}
qat_priv = dev->data->dev_private;
qat_stats_get(qat_priv->qat_dev, &qat_stats, QAT_SERVICE_ASYMMETRIC);
stats->enqueued_count = qat_stats.enqueued_count;
stats->dequeued_count = qat_stats.dequeued_count;
stats->enqueue_err_count = qat_stats.enqueue_err_count;
stats->dequeue_err_count = qat_stats.dequeue_err_count;
}
static void qat_asym_stats_reset(struct rte_cryptodev *dev)
{
struct qat_asym_dev_private *qat_priv;
if (dev == NULL) {
QAT_LOG(ERR, "invalid asymmetric cryptodev ptr %p", dev);
return;
}
qat_priv = dev->data->dev_private;
qat_stats_reset(qat_priv->qat_dev, QAT_SERVICE_ASYMMETRIC);
}
static int qat_asym_qp_release(struct rte_cryptodev *dev,
uint16_t queue_pair_id)
{
struct qat_asym_dev_private *qat_private = dev->data->dev_private;
enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen;
QAT_LOG(DEBUG, "Release asym qp %u on device %d",
queue_pair_id, dev->data->dev_id);
qat_private->qat_dev->qps_in_use[QAT_SERVICE_ASYMMETRIC][queue_pair_id]
= NULL;
return qat_qp_release(qat_dev_gen, (struct qat_qp **)
&(dev->data->queue_pairs[queue_pair_id]));
}
static int qat_asym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
const struct rte_cryptodev_qp_conf *qp_conf,
int socket_id)
{
struct qat_qp_config qat_qp_conf;
struct qat_qp *qp;
int ret = 0;
uint32_t i;
struct qat_qp **qp_addr =
(struct qat_qp **)&(dev->data->queue_pairs[qp_id]);
struct qat_asym_dev_private *qat_private = dev->data->dev_private;
struct qat_pci_device *qat_dev = qat_private->qat_dev;
const struct qat_qp_hw_data *asym_hw_qps =
qat_gen_config[qat_private->qat_dev->qat_dev_gen]
.qp_hw_data[QAT_SERVICE_ASYMMETRIC];
const struct qat_qp_hw_data *qp_hw_data = asym_hw_qps + qp_id;
/* If qp is already in use free ring memory and qp metadata. */
if (*qp_addr != NULL) {
ret = qat_asym_qp_release(dev, qp_id);
if (ret < 0)
return ret;
}
if (qp_id >= qat_qps_per_service(qat_dev, QAT_SERVICE_ASYMMETRIC)) {
QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id);
return -EINVAL;
}
qat_qp_conf.hw = qp_hw_data;
qat_qp_conf.cookie_size = sizeof(struct qat_asym_op_cookie);
qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
qat_qp_conf.socket_id = socket_id;
qat_qp_conf.service_str = "asym";
ret = qat_qp_setup(qat_private->qat_dev, qp_addr, qp_id, &qat_qp_conf);
if (ret != 0)
return ret;
/* store a link to the qp in the qat_pci_device */
qat_private->qat_dev->qps_in_use[QAT_SERVICE_ASYMMETRIC][qp_id]
= *qp_addr;
qp = (struct qat_qp *)*qp_addr;
qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold;
for (i = 0; i < qp->nb_descriptors; i++) {
int j;
struct qat_asym_op_cookie __rte_unused *cookie =
qp->op_cookies[i];
cookie->input_addr = rte_mempool_virt2iova(cookie) +
for (j = 0; j < 8; j++) {
cookie->input_params_ptrs[j] =
rte_mempool_virt2iova(cookie) +
offsetof(struct qat_asym_op_cookie,
input_params_ptrs);
cookie->output_addr = rte_mempool_virt2iova(cookie) +
input_array[j]);
cookie->output_params_ptrs[j] =
rte_mempool_virt2iova(cookie) +
offsetof(struct qat_asym_op_cookie,
output_params_ptrs);
for (j = 0; j < 8; j++) {
cookie->input_params_ptrs[j] =
rte_mempool_virt2iova(cookie) +
offsetof(struct qat_asym_op_cookie,
input_array[j]);
cookie->output_params_ptrs[j] =
rte_mempool_virt2iova(cookie) +
offsetof(struct qat_asym_op_cookie,
output_array[j]);
}
output_array[j]);
}
return ret;
}
struct rte_cryptodev_ops crypto_qat_ops = {
static struct rte_cryptodev_ops crypto_qat_ops = {
/* Device related operations */
.dev_configure = qat_asym_dev_config,
.dev_start = qat_asym_dev_start,
.dev_stop = qat_asym_dev_stop,
.dev_close = qat_asym_dev_close,
.dev_infos_get = qat_asym_dev_info_get,
.dev_configure = qat_cryptodev_config,
.dev_start = qat_cryptodev_start,
.dev_stop = qat_cryptodev_stop,
.dev_close = qat_cryptodev_close,
.dev_infos_get = qat_cryptodev_info_get,
.stats_get = qat_asym_stats_get,
.stats_reset = qat_asym_stats_reset,
.queue_pair_setup = qat_asym_qp_setup,
.queue_pair_release = qat_asym_qp_release,
.stats_get = qat_cryptodev_stats_get,
.stats_reset = qat_cryptodev_stats_reset,
.queue_pair_setup = qat_cryptodev_qp_setup,
.queue_pair_release = qat_cryptodev_qp_release,
/* Crypto related operations */
.asym_session_get_size = qat_asym_session_get_private_size,
@ -241,15 +97,14 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
struct qat_device_info *qat_dev_instance =
&qat_pci_devs[qat_pci_dev->qat_dev_id];
struct rte_cryptodev_pmd_init_params init_params = {
.name = "",
.socket_id =
qat_dev_instance->pci_dev->device.numa_node,
.private_data_size = sizeof(struct qat_asym_dev_private)
.name = "",
.socket_id = qat_dev_instance->pci_dev->device.numa_node,
.private_data_size = sizeof(struct qat_cryptodev_private)
};
char name[RTE_CRYPTODEV_NAME_MAX_LEN];
char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
struct rte_cryptodev *cryptodev;
struct qat_asym_dev_private *internals;
struct qat_cryptodev_private *internals;
if (qat_pci_dev->qat_dev_gen == QAT_GEN4) {
QAT_LOG(ERR, "Asymmetric crypto PMD not supported on QAT 4xxx");
@ -310,8 +165,9 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
internals = cryptodev->data->dev_private;
internals->qat_dev = qat_pci_dev;
internals->asym_dev_id = cryptodev->data->dev_id;
internals->dev_id = cryptodev->data->dev_id;
internals->qat_dev_capabilities = qat_gen1_asym_capabilities;
internals->service_type = QAT_SERVICE_ASYMMETRIC;
internals->capa_mz = rte_memzone_lookup(capa_memz_name);
if (internals->capa_mz == NULL) {
@ -347,7 +203,7 @@ qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
rte_cryptodev_pmd_probing_finish(cryptodev);
QAT_LOG(DEBUG, "Created QAT ASYM device %s as cryptodev instance %d",
cryptodev->data->name, internals->asym_dev_id);
cryptodev->data->name, internals->dev_id);
return 0;
}
@ -365,7 +221,7 @@ qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev)
/* free crypto device */
cryptodev = rte_cryptodev_pmd_get_dev(
qat_pci_dev->asym_dev->asym_dev_id);
qat_pci_dev->asym_dev->dev_id);
rte_cryptodev_pmd_destroy(cryptodev);
qat_pci_devs[qat_pci_dev->qat_dev_id].asym_rte_dev.name = NULL;
qat_pci_dev->asym_dev = NULL;

29
drivers/crypto/qat/qat_asym_pmd.h
View File

@ -15,21 +15,8 @@
extern uint8_t qat_asym_driver_id;
/** private data structure for a QAT device.
* This QAT device is a device offering only asymmetric crypto service,
* there can be one of these on each qat_pci_device (VF).
*/
struct qat_asym_dev_private {
struct qat_pci_device *qat_dev;
/**< The qat pci device hosting the service */
uint8_t asym_dev_id;
/**< Device instance for this rte_cryptodev */
const struct rte_cryptodev_capabilities *qat_dev_capabilities;
/* QAT device asymmetric crypto capabilities */
const struct rte_memzone *capa_mz;
/* Shared memzone for storing capabilities */
uint16_t min_enq_burst_threshold;
};
void
qat_asym_init_op_cookie(void *op_cookie);
uint16_t
qat_asym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
@ -39,16 +26,4 @@ uint16_t
qat_asym_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
uint16_t nb_ops);
int qat_asym_session_configure(struct rte_cryptodev *dev,
struct rte_crypto_asym_xform *xform,
struct rte_cryptodev_asym_session *sess,
struct rte_mempool *mempool);
int
qat_asym_dev_create(struct qat_pci_device *qat_pci_dev,
struct qat_dev_cmd_param *qat_dev_cmd_param);
int
qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev);
#endif /* _QAT_ASYM_PMD_H_ */

176
drivers/crypto/qat/qat_crypto.c Normal file
View File

@ -0,0 +1,176 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2021 Intel Corporation
*/
#include "qat_device.h"
#include "qat_qp.h"
#include "qat_crypto.h"
#include "qat_sym.h"
#include "qat_asym.h"
int
qat_cryptodev_config(__rte_unused struct rte_cryptodev *dev,
__rte_unused struct rte_cryptodev_config *config)
{
return 0;
}
int
qat_cryptodev_start(__rte_unused struct rte_cryptodev *dev)
{
return 0;
}
void
qat_cryptodev_stop(__rte_unused struct rte_cryptodev *dev)
{
}
int
qat_cryptodev_close(struct rte_cryptodev *dev)
{
int i, ret;
for (i = 0; i < dev->data->nb_queue_pairs; i++) {
ret = dev->dev_ops->queue_pair_release(dev, i);
if (ret < 0)
return ret;
}
return 0;
}
void
qat_cryptodev_info_get(struct rte_cryptodev *dev,
struct rte_cryptodev_info *info)
{
struct qat_cryptodev_private *qat_private = dev->data->dev_private;
struct qat_pci_device *qat_dev = qat_private->qat_dev;
enum qat_service_type service_type = qat_private->service_type;
if (info != NULL) {
info->max_nb_queue_pairs =
qat_qps_per_service(qat_dev, service_type);
info->feature_flags = dev->feature_flags;
info->capabilities = qat_private->qat_dev_capabilities;
if (service_type == QAT_SERVICE_ASYMMETRIC)
info->driver_id = qat_asym_driver_id;
if (service_type == QAT_SERVICE_SYMMETRIC)
info->driver_id = qat_sym_driver_id;
/* No limit of number of sessions */
info->sym.max_nb_sessions = 0;
}
}
void
qat_cryptodev_stats_get(struct rte_cryptodev *dev,
struct rte_cryptodev_stats *stats)
{
struct qat_common_stats qat_stats = {0};
struct qat_cryptodev_private *qat_priv;
if (stats == NULL || dev == NULL) {
QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev);
return;
}
qat_priv = dev->data->dev_private;
qat_stats_get(qat_priv->qat_dev, &qat_stats, qat_priv->service_type);
stats->enqueued_count = qat_stats.enqueued_count;
stats->dequeued_count = qat_stats.dequeued_count;
stats->enqueue_err_count = qat_stats.enqueue_err_count;
stats->dequeue_err_count = qat_stats.dequeue_err_count;
}
void
qat_cryptodev_stats_reset(struct rte_cryptodev *dev)
{
struct qat_cryptodev_private *qat_priv;
if (dev == NULL) {
QAT_LOG(ERR, "invalid cryptodev ptr %p", dev);
return;
}
qat_priv = dev->data->dev_private;
qat_stats_reset(qat_priv->qat_dev, qat_priv->service_type);
}
int
qat_cryptodev_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
{
struct qat_cryptodev_private *qat_private = dev->data->dev_private;
struct qat_pci_device *qat_dev = qat_private->qat_dev;
enum qat_device_gen qat_dev_gen = qat_dev->qat_dev_gen;
enum qat_service_type service_type = qat_private->service_type;
QAT_LOG(DEBUG, "Release %s qp %u on device %d",
qat_service_get_str(service_type),
queue_pair_id, dev->data->dev_id);
qat_private->qat_dev->qps_in_use[service_type][queue_pair_id] = NULL;
return qat_qp_release(qat_dev_gen, (struct qat_qp **)
&(dev->data->queue_pairs[queue_pair_id]));
}
int
qat_cryptodev_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
const struct rte_cryptodev_qp_conf *qp_conf, int socket_id)
{
struct qat_qp **qp_addr =
(struct qat_qp **)&(dev->data->queue_pairs[qp_id]);
struct qat_cryptodev_private *qat_private = dev->data->dev_private;
struct qat_pci_device *qat_dev = qat_private->qat_dev;
enum qat_service_type service_type = qat_private->service_type;
struct qat_qp_config qat_qp_conf = {0};
struct qat_qp *qp;
int ret = 0;
uint32_t i;
/* If qp is already in use free ring memory and qp metadata. */
if (*qp_addr != NULL) {
ret = dev->dev_ops->queue_pair_release(dev, qp_id);
if (ret < 0)
return -EBUSY;
}
if (qp_id >= qat_qps_per_service(qat_dev, service_type)) {
QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id);
return -EINVAL;
}
qat_qp_conf.hw = qat_qp_get_hw_data(qat_dev, service_type,
qp_id);
if (qat_qp_conf.hw == NULL) {
QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id);
return -EINVAL;
}
qat_qp_conf.cookie_size = service_type == QAT_SERVICE_SYMMETRIC ?
sizeof(struct qat_sym_op_cookie) :
sizeof(struct qat_asym_op_cookie);
qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
qat_qp_conf.socket_id = socket_id;
qat_qp_conf.service_str = qat_service_get_str(service_type);
ret = qat_qp_setup(qat_dev, qp_addr, qp_id, &qat_qp_conf);
if (ret != 0)
return ret;
/* store a link to the qp in the qat_pci_device */
qat_dev->qps_in_use[service_type][qp_id] = *qp_addr;
qp = (struct qat_qp *)*qp_addr;
qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold;
for (i = 0; i < qp->nb_descriptors; i++) {
if (service_type == QAT_SERVICE_SYMMETRIC)
qat_sym_init_op_cookie(qp->op_cookies[i]);
else
qat_asym_init_op_cookie(qp->op_cookies[i]);
}
return ret;
}

78
drivers/crypto/qat/qat_crypto.h Normal file
View File

@ -0,0 +1,78 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2021 Intel Corporation
*/
#ifndef _QAT_CRYPTO_H_
#define _QAT_CRYPTO_H_
#include <rte_cryptodev.h>
#ifdef RTE_LIB_SECURITY
#include <rte_security.h>
#endif
#include "qat_device.h"
extern uint8_t qat_sym_driver_id;
extern uint8_t qat_asym_driver_id;
/** helper macro to set cryptodev capability range **/
#define CAP_RNG(n, l, r, i) .n = {.min = l, .max = r, .increment = i}
#define CAP_RNG_ZERO(n) .n = {.min = 0, .max = 0, .increment = 0}
/** helper macro to set cryptodev capability value **/
#define CAP_SET(n, v) .n = v
/** private data structure for a QAT device.
* there can be one of these on each qat_pci_device (VF).
*/
struct qat_cryptodev_private {
struct qat_pci_device *qat_dev;
/**< The qat pci device hosting the service */
uint8_t dev_id;
/**< Device instance for this rte_cryptodev */
const struct rte_cryptodev_capabilities *qat_dev_capabilities;
/* QAT device symmetric crypto capabilities */
const struct rte_memzone *capa_mz;
/* Shared memzone for storing capabilities */
uint16_t min_enq_burst_threshold;
uint32_t internal_capabilities; /* see flags QAT_SYM_CAP_xxx */
enum qat_service_type service_type;
};
struct qat_capabilities_info {
struct rte_cryptodev_capabilities *data;
uint64_t size;
};
int
qat_cryptodev_config(struct rte_cryptodev *dev,
struct rte_cryptodev_config *config);
int
qat_cryptodev_start(struct rte_cryptodev *dev);
void
qat_cryptodev_stop(struct rte_cryptodev *dev);
int
qat_cryptodev_close(struct rte_cryptodev *dev);
void
qat_cryptodev_info_get(struct rte_cryptodev *dev,
struct rte_cryptodev_info *info);
void
qat_cryptodev_stats_get(struct rte_cryptodev *dev,
struct rte_cryptodev_stats *stats);
void
qat_cryptodev_stats_reset(struct rte_cryptodev *dev);
int
qat_cryptodev_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
const struct rte_cryptodev_qp_conf *qp_conf, int socket_id);
int
qat_cryptodev_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id);
#endif

250
drivers/crypto/qat/qat_sym_pmd.c
View File

@ -13,6 +13,7 @@
#endif
#include "qat_logs.h"
#include "qat_crypto.h"
#include "qat_sym.h"
#include "qat_sym_session.h"
#include "qat_sym_pmd.h"
@ -59,213 +60,19 @@ static const struct rte_security_capability qat_security_capabilities[] = {
};
#endif
static int qat_sym_qp_release(struct rte_cryptodev *dev,
uint16_t queue_pair_id);
static int qat_sym_dev_config(__rte_unused struct rte_cryptodev *dev,
__rte_unused struct rte_cryptodev_config *config)
{
return 0;
}
static int qat_sym_dev_start(__rte_unused struct rte_cryptodev *dev)
{
return 0;
}
static void qat_sym_dev_stop(__rte_unused struct rte_cryptodev *dev)
{
return;
}
static int qat_sym_dev_close(struct rte_cryptodev *dev)
{
int i, ret;
for (i = 0; i < dev->data->nb_queue_pairs; i++) {
ret = qat_sym_qp_release(dev, i);
if (ret < 0)
return ret;
}
return 0;
}
static void qat_sym_dev_info_get(struct rte_cryptodev *dev,
struct rte_cryptodev_info *info)
{
struct qat_sym_dev_private *internals = dev->data->dev_private;
struct qat_pci_device *qat_dev = internals->qat_dev;
if (info != NULL) {
info->max_nb_queue_pairs =
qat_qps_per_service(qat_dev, QAT_SERVICE_SYMMETRIC);
info->feature_flags = dev->feature_flags;
info->capabilities = internals->qat_dev_capabilities;
info->driver_id = qat_sym_driver_id;
/* No limit of number of sessions */
info->sym.max_nb_sessions = 0;
}
}
static void qat_sym_stats_get(struct rte_cryptodev *dev,
struct rte_cryptodev_stats *stats)
{
struct qat_common_stats qat_stats = {0};
struct qat_sym_dev_private *qat_priv;
if (stats == NULL || dev == NULL) {
QAT_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev);
return;
}
qat_priv = dev->data->dev_private;
qat_stats_get(qat_priv->qat_dev, &qat_stats, QAT_SERVICE_SYMMETRIC);
stats->enqueued_count = qat_stats.enqueued_count;
stats->dequeued_count = qat_stats.dequeued_count;
stats->enqueue_err_count = qat_stats.enqueue_err_count;
stats->dequeue_err_count = qat_stats.dequeue_err_count;
}
static void qat_sym_stats_reset(struct rte_cryptodev *dev)
{
struct qat_sym_dev_private *qat_priv;
if (dev == NULL) {
QAT_LOG(ERR, "invalid cryptodev ptr %p", dev);
return;
}
qat_priv = dev->data->dev_private;
qat_stats_reset(qat_priv->qat_dev, QAT_SERVICE_SYMMETRIC);
}
static int qat_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
{
struct qat_sym_dev_private *qat_private = dev->data->dev_private;
enum qat_device_gen qat_dev_gen = qat_private->qat_dev->qat_dev_gen;
QAT_LOG(DEBUG, "Release sym qp %u on device %d",
queue_pair_id, dev->data->dev_id);
qat_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][queue_pair_id]
= NULL;
return qat_qp_release(qat_dev_gen, (struct qat_qp **)
&(dev->data->queue_pairs[queue_pair_id]));
}
static int qat_sym_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
const struct rte_cryptodev_qp_conf *qp_conf,
int socket_id)
{
struct qat_qp *qp;
int ret = 0;
uint32_t i;
struct qat_qp_config qat_qp_conf;
struct qat_qp **qp_addr =
(struct qat_qp **)&(dev->data->queue_pairs[qp_id]);
struct qat_sym_dev_private *qat_private = dev->data->dev_private;
struct qat_pci_device *qat_dev = qat_private->qat_dev;
/* If qp is already in use free ring memory and qp metadata. */
if (*qp_addr != NULL) {
ret = qat_sym_qp_release(dev, qp_id);
if (ret < 0)
return ret;
}
if (qp_id >= qat_qps_per_service(qat_dev, QAT_SERVICE_SYMMETRIC)) {
QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id);
return -EINVAL;
}
qat_qp_conf.hw = qat_qp_get_hw_data(qat_dev, QAT_SERVICE_SYMMETRIC,
qp_id);
if (qat_qp_conf.hw == NULL) {
QAT_LOG(ERR, "qp_id %u invalid for this device", qp_id);
return -EINVAL;
}
qat_qp_conf.cookie_size = sizeof(struct qat_sym_op_cookie);
qat_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
qat_qp_conf.socket_id = socket_id;
qat_qp_conf.service_str = "sym";
ret = qat_qp_setup(qat_private->qat_dev, qp_addr, qp_id, &qat_qp_conf);
if (ret != 0)
return ret;
/* store a link to the qp in the qat_pci_device */
qat_private->qat_dev->qps_in_use[QAT_SERVICE_SYMMETRIC][qp_id]
= *qp_addr;
qp = (struct qat_qp *)*qp_addr;
qp->min_enq_burst_threshold = qat_private->min_enq_burst_threshold;
for (i = 0; i < qp->nb_descriptors; i++) {
struct qat_sym_op_cookie *cookie =
qp->op_cookies[i];
cookie->qat_sgl_src_phys_addr =
rte_mempool_virt2iova(cookie) +
offsetof(struct qat_sym_op_cookie,
qat_sgl_src);
cookie->qat_sgl_dst_phys_addr =
rte_mempool_virt2iova(cookie) +
offsetof(struct qat_sym_op_cookie,
qat_sgl_dst);
cookie->opt.spc_gmac.cd_phys_addr =
rte_mempool_virt2iova(cookie) +
offsetof(struct qat_sym_op_cookie,
opt.spc_gmac.cd_cipher);
}
/* Get fw version from QAT (GEN2), skip if we've got it already */
if (qp->qat_dev_gen == QAT_GEN2 && !(qat_private->internal_capabilities
& QAT_SYM_CAP_VALID)) {
ret = qat_cq_get_fw_version(qp);
if (ret < 0) {
qat_sym_qp_release(dev, qp_id);
return ret;
}
if (ret != 0)
QAT_LOG(DEBUG, "QAT firmware version: %d.%d.%d",
(ret >> 24) & 0xff,
(ret >> 16) & 0xff,
(ret >> 8) & 0xff);
else
QAT_LOG(DEBUG, "unknown QAT firmware version");
/* set capabilities based on the fw version */
qat_private->internal_capabilities = QAT_SYM_CAP_VALID |
((ret >= MIXED_CRYPTO_MIN_FW_VER) ?
QAT_SYM_CAP_MIXED_CRYPTO : 0);
ret = 0;
}
return ret;
}
static struct rte_cryptodev_ops crypto_qat_ops = {
/* Device related operations */
.dev_configure = qat_sym_dev_config,
.dev_start = qat_sym_dev_start,
.dev_stop = qat_sym_dev_stop,
.dev_close = qat_sym_dev_close,
.dev_infos_get = qat_sym_dev_info_get,
.dev_configure = qat_cryptodev_config,
.dev_start = qat_cryptodev_start,
.dev_stop = qat_cryptodev_stop,
.dev_close = qat_cryptodev_close,
.dev_infos_get = qat_cryptodev_info_get,
.stats_get = qat_sym_stats_get,
.stats_reset = qat_sym_stats_reset,
.queue_pair_setup = qat_sym_qp_setup,
.queue_pair_release = qat_sym_qp_release,
.stats_get = qat_cryptodev_stats_get,
.stats_reset = qat_cryptodev_stats_reset,
.queue_pair_setup = qat_cryptodev_qp_setup,
.queue_pair_release = qat_cryptodev_qp_release,
/* Crypto related operations */
.sym_session_get_size = qat_sym_session_get_private_size,
@ -295,6 +102,27 @@ static struct rte_security_ops security_qat_ops = {
};
#endif
void
qat_sym_init_op_cookie(void *op_cookie)
{
struct qat_sym_op_cookie *cookie = op_cookie;
cookie->qat_sgl_src_phys_addr =
rte_mempool_virt2iova(cookie) +
offsetof(struct qat_sym_op_cookie,
qat_sgl_src);
cookie->qat_sgl_dst_phys_addr =
rte_mempool_virt2iova(cookie) +
offsetof(struct qat_sym_op_cookie,
qat_sgl_dst);
cookie->opt.spc_gmac.cd_phys_addr =
rte_mempool_virt2iova(cookie) +
offsetof(struct qat_sym_op_cookie,
opt.spc_gmac.cd_cipher);
}
static uint16_t
qat_sym_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
uint16_t nb_ops)
@ -330,15 +158,14 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
&qat_pci_devs[qat_pci_dev->qat_dev_id];
struct rte_cryptodev_pmd_init_params init_params = {
.name = "",
.socket_id =
qat_dev_instance->pci_dev->device.numa_node,
.private_data_size = sizeof(struct qat_sym_dev_private)
.name = "",
.socket_id = qat_dev_instance->pci_dev->device.numa_node,
.private_data_size = sizeof(struct qat_cryptodev_private)
};
char name[RTE_CRYPTODEV_NAME_MAX_LEN];
char capa_memz_name[RTE_CRYPTODEV_NAME_MAX_LEN];
struct rte_cryptodev *cryptodev;
struct qat_sym_dev_private *internals;
struct qat_cryptodev_private *internals;
const struct rte_cryptodev_capabilities *capabilities;
uint64_t capa_size;
@ -424,8 +251,9 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
internals = cryptodev->data->dev_private;
internals->qat_dev = qat_pci_dev;
internals->service_type = QAT_SERVICE_SYMMETRIC;
internals->sym_dev_id = cryptodev->data->dev_id;
internals->dev_id = cryptodev->data->dev_id;
switch (qat_pci_dev->qat_dev_gen) {
case QAT_GEN1:
capabilities = qat_gen1_sym_capabilities;
@ -480,7 +308,7 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
qat_pci_dev->sym_dev = internals;
QAT_LOG(DEBUG, "Created QAT SYM device %s as cryptodev instance %d",
cryptodev->data->name, internals->sym_dev_id);
cryptodev->data->name, internals->dev_id);
rte_cryptodev_pmd_probing_finish(cryptodev);
@ -511,7 +339,7 @@ qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev)
rte_memzone_free(qat_pci_dev->sym_dev->capa_mz);
/* free crypto device */
cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->sym_dev_id);
cryptodev = rte_cryptodev_pmd_get_dev(qat_pci_dev->sym_dev->dev_id);
#ifdef RTE_LIB_SECURITY
rte_free(cryptodev->security_ctx);
cryptodev->security_ctx = NULL;

21
drivers/crypto/qat/qat_sym_pmd.h
View File

@ -14,6 +14,7 @@
#endif
#include "qat_sym_capabilities.h"
#include "qat_crypto.h"
#include "qat_device.h"
/** Intel(R) QAT Symmetric Crypto PMD driver name */
@ -25,23 +26,6 @@
extern uint8_t qat_sym_driver_id;
/** private data structure for a QAT device.
* This QAT device is a device offering only symmetric crypto service,
* there can be one of these on each qat_pci_device (VF).
*/
struct qat_sym_dev_private {
struct qat_pci_device *qat_dev;
/**< The qat pci device hosting the service */
uint8_t sym_dev_id;
/**< Device instance for this rte_cryptodev */
const struct rte_cryptodev_capabilities *qat_dev_capabilities;
/* QAT device symmetric crypto capabilities */
const struct rte_memzone *capa_mz;
/* Shared memzone for storing capabilities */
uint16_t min_enq_burst_threshold;
uint32_t internal_capabilities; /* see flags QAT_SYM_CAP_xxx */
};
int
qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
struct qat_dev_cmd_param *qat_dev_cmd_param);
@ -49,5 +33,8 @@ qat_sym_dev_create(struct qat_pci_device *qat_pci_dev,
int
qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev);
void
qat_sym_init_op_cookie(void *op_cookie);
#endif
#endif /* _QAT_SYM_PMD_H_ */

15
drivers/crypto/qat/qat_sym_session.c
View File

@ -131,7 +131,7 @@ bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo,
static int
qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo,
struct qat_sym_dev_private *internals)
struct qat_cryptodev_private *internals)
{
int i = 0;
const struct rte_cryptodev_capabilities *capability;
@ -152,7 +152,7 @@ qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo,
static int
qat_is_auth_alg_supported(enum rte_crypto_auth_algorithm algo,
struct qat_sym_dev_private *internals)
struct qat_cryptodev_private *internals)
{
int i = 0;
const struct rte_cryptodev_capabilities *capability;
@ -267,7 +267,7 @@ qat_sym_session_configure_cipher(struct rte_cryptodev *dev,
struct rte_crypto_sym_xform *xform,
struct qat_sym_session *session)
{
struct qat_sym_dev_private *internals = dev->data->dev_private;
struct qat_cryptodev_private *internals = dev->data->dev_private;
struct rte_crypto_cipher_xform *cipher_xform = NULL;
enum qat_device_gen qat_dev_gen =
internals->qat_dev->qat_dev_gen;
@ -532,7 +532,8 @@ static void
qat_sym_session_handle_mixed(const struct rte_cryptodev *dev,
struct qat_sym_session *session)
{
const struct qat_sym_dev_private *qat_private = dev->data->dev_private;
const struct qat_cryptodev_private *qat_private =
dev->data->dev_private;
enum qat_device_gen min_dev_gen = (qat_private->internal_capabilities &
QAT_SYM_CAP_MIXED_CRYPTO) ? QAT_GEN2 : QAT_GEN3;
@ -564,7 +565,7 @@ qat_sym_session_set_parameters(struct rte_cryptodev *dev,
struct rte_crypto_sym_xform *xform, void *session_private)
{
struct qat_sym_session *session = session_private;
struct qat_sym_dev_private *internals = dev->data->dev_private;
struct qat_cryptodev_private *internals = dev->data->dev_private;
enum qat_device_gen qat_dev_gen = internals->qat_dev->qat_dev_gen;
int ret;
int qat_cmd_id;
@ -707,7 +708,7 @@ qat_sym_session_configure_auth(struct rte_cryptodev *dev,
struct qat_sym_session *session)
{
struct rte_crypto_auth_xform *auth_xform = qat_get_auth_xform(xform);
struct qat_sym_dev_private *internals = dev->data->dev_private;
struct qat_cryptodev_private *internals = dev->data->dev_private;
const uint8_t *key_data = auth_xform->key.data;
uint8_t key_length = auth_xform->key.length;
enum qat_device_gen qat_dev_gen =
@ -875,7 +876,7 @@ qat_sym_session_configure_aead(struct rte_cryptodev *dev,
{
struct rte_crypto_aead_xform *aead_xform = &xform->aead;
enum rte_crypto_auth_operation crypto_operation;
struct qat_sym_dev_private *internals =
struct qat_cryptodev_private *internals =
dev->data->dev_private;
enum qat_device_gen qat_dev_gen =
internals->qat_dev->qat_dev_gen;