crypto/bcmfs: create a symmetric cryptodev

Create a symmetric crypto device and add supported cryptodev ops.

Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
Signed-off-by: Raveendra Padasalagi <raveendra.padasalagi@broadcom.com>
Reviewed-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
Acked-by: Akhil Goyal <akhil.goyal@nxp.com>

drivers/crypto/bcmfs/bcmfs_device.c

@@ -14,6 +14,7 @@
 #include "bcmfs_logs.h"
 #include "bcmfs_qp.h"
 #include "bcmfs_vfio.h"
+#include "bcmfs_sym_pmd.h"
 
 struct bcmfs_device_attr {
 	const char name[BCMFS_MAX_PATH_LEN];
@@ -240,6 +241,7 @@ bcmfs_vdev_probe(struct rte_vdev_device *vdev)
 	char out_dirname[BCMFS_MAX_PATH_LEN];
 	uint32_t fsdev_dev[BCMFS_MAX_NODES];
 	enum bcmfs_device_type dtype;
+	int err;
 	int i = 0;
 	int dev_idx;
 	int count = 0;
@@ -291,7 +293,20 @@ bcmfs_vdev_probe(struct rte_vdev_device *vdev)
 		return -ENODEV;
 	}
 
+	err = bcmfs_sym_dev_create(fsdev);
+	if (err) {
+		BCMFS_LOG(WARNING,
+			  "Failed to create BCMFS SYM PMD for device %s",
+			  fsdev->name);
+		goto pmd_create_fail;
+	}
+
 	return 0;
+
+pmd_create_fail:
+	fsdev_release(fsdev);
+
+	return err;
 }
 
 static int

drivers/crypto/bcmfs/bcmfs_device.h

@@ -65,6 +65,12 @@ struct bcmfs_device {
 	struct bcmfs_qp *qps_in_use[BCMFS_MAX_HW_QUEUES];
 	/* queue pair ops exported by symmetric crypto hw */
 	struct bcmfs_hw_queue_pair_ops *sym_hw_qp_ops;
+	/* a cryptodevice attached to bcmfs device */
+	struct rte_cryptodev *cdev;
+	/* a rte_device to register with cryptodev */
+	struct rte_device sym_rte_dev;
+	/* private info to keep with cryptodev */
+	struct bcmfs_sym_dev_private *sym_dev;
 };
 
 #endif /* _BCMFS_DEVICE_H_ */

drivers/crypto/bcmfs/bcmfs_qp.c

@@ -344,3 +344,40 @@ bcmfs_dequeue_op_burst(void *qp, void **ops, uint16_t nb_ops)
 
 	return deq;
 }
+
+void bcmfs_qp_stats_get(struct bcmfs_qp **qp, int num_qp,
+			struct bcmfs_qp_stats *stats)
+{
+	int i;
+
+	if (stats == NULL) {
+		BCMFS_LOG(ERR, "invalid param: stats %p",
+			  stats);
+		return;
+	}
+
+	for (i = 0; i < num_qp; i++) {
+		if (qp[i] == NULL) {
+			BCMFS_LOG(DEBUG, "Uninitialised qp %d", i);
+			continue;
+		}
+
+		stats->enqueued_count += qp[i]->stats.enqueued_count;
+		stats->dequeued_count += qp[i]->stats.dequeued_count;
+		stats->enqueue_err_count += qp[i]->stats.enqueue_err_count;
+		stats->dequeue_err_count += qp[i]->stats.dequeue_err_count;
+	}
+}
+
+void bcmfs_qp_stats_reset(struct bcmfs_qp **qp, int num_qp)
+{
+	int i;
+
+	for (i = 0; i < num_qp; i++) {
+		if (qp[i] == NULL) {
+			BCMFS_LOG(DEBUG, "Uninitialised qp %d", i);
+			continue;
+		}
+		memset(&qp[i]->stats, 0, sizeof(qp[i]->stats));
+	}
+}

drivers/crypto/bcmfs/bcmfs_qp.h

@@ -24,6 +24,13 @@ enum bcmfs_queue_type {
 	BCMFS_RM_CPLQ
 };
 
+#define BCMFS_QP_IOBASE_XLATE(base, idx)	\
+		((base) + ((idx) * BCMFS_HW_QUEUE_IO_ADDR_LEN))
+
+/* Max pkts for preprocessing before submitting to h/w qp */
+#define BCMFS_MAX_REQS_BUFF	64
+
+/* qp stats */
 struct bcmfs_qp_stats {
 	/* Count of all operations enqueued */
 	uint64_t enqueued_count;
@@ -92,6 +99,10 @@ struct bcmfs_qp {
 	struct bcmfs_qp_stats stats;
 	/* h/w ops associated with qp */
 	struct bcmfs_hw_queue_pair_ops *ops;
+	/* bcmfs requests pool*/
+	struct rte_mempool *sr_mp;
+	/* a temporary buffer to keep message pointers */
+	struct bcmfs_qp_message *infl_msgs[BCMFS_MAX_REQS_BUFF];
 
 } __rte_cache_aligned;
 
@@ -123,4 +134,9 @@ bcmfs_qp_setup(struct bcmfs_qp **qp_addr,
 	       uint16_t queue_pair_id,
 	       struct bcmfs_qp_config *bcmfs_conf);
 
+/* stats functions*/
+void bcmfs_qp_stats_get(struct bcmfs_qp **qp, int num_qp,
+			struct bcmfs_qp_stats *stats);
+void bcmfs_qp_stats_reset(struct bcmfs_qp **qp, int num_qp);
+
 #endif /* _BCMFS_QP_H_ */

drivers/crypto/bcmfs/bcmfs_sym_pmd.c

@@ -0,0 +1,387 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Broadcom
+ * All rights reserved.
+ */
+
+#include <rte_common.h>
+#include <rte_dev.h>
+#include <rte_errno.h>
+#include <rte_malloc.h>
+#include <rte_cryptodev_pmd.h>
+
+#include "bcmfs_device.h"
+#include "bcmfs_logs.h"
+#include "bcmfs_qp.h"
+#include "bcmfs_sym_pmd.h"
+#include "bcmfs_sym_req.h"
+
+uint8_t cryptodev_bcmfs_driver_id;
+
+static int bcmfs_sym_qp_release(struct rte_cryptodev *dev,
+				uint16_t queue_pair_id);
+
+static int
+bcmfs_sym_dev_config(__rte_unused struct rte_cryptodev *dev,
+		     __rte_unused struct rte_cryptodev_config *config)
+{
+	return 0;
+}
+
+static int
+bcmfs_sym_dev_start(__rte_unused struct rte_cryptodev *dev)
+{
+	return 0;
+}
+
+static void
+bcmfs_sym_dev_stop(__rte_unused struct rte_cryptodev *dev)
+{
+}
+
+static int
+bcmfs_sym_dev_close(struct rte_cryptodev *dev)
+{
+	int i, ret;
+
+	for (i = 0; i < dev->data->nb_queue_pairs; i++) {
+		ret = bcmfs_sym_qp_release(dev, i);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void
+bcmfs_sym_dev_info_get(struct rte_cryptodev *dev,
+		       struct rte_cryptodev_info *dev_info)
+{
+	struct bcmfs_sym_dev_private *internals = dev->data->dev_private;
+	struct bcmfs_device *fsdev = internals->fsdev;
+
+	if (dev_info != NULL) {
+		dev_info->driver_id = cryptodev_bcmfs_driver_id;
+		dev_info->feature_flags = dev->feature_flags;
+		dev_info->max_nb_queue_pairs = fsdev->max_hw_qps;
+		/* No limit of number of sessions */
+		dev_info->sym.max_nb_sessions = 0;
+	}
+}
+
+static void
+bcmfs_sym_stats_get(struct rte_cryptodev *dev,
+		    struct rte_cryptodev_stats *stats)
+{
+	struct bcmfs_qp_stats bcmfs_stats = {0};
+	struct bcmfs_sym_dev_private *bcmfs_priv;
+	struct bcmfs_device *fsdev;
+
+	if (stats == NULL || dev == NULL) {
+		BCMFS_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev);
+		return;
+	}
+	bcmfs_priv = dev->data->dev_private;
+	fsdev = bcmfs_priv->fsdev;
+
+	bcmfs_qp_stats_get(fsdev->qps_in_use, fsdev->max_hw_qps, &bcmfs_stats);
+
+	stats->enqueued_count = bcmfs_stats.enqueued_count;
+	stats->dequeued_count = bcmfs_stats.dequeued_count;
+	stats->enqueue_err_count = bcmfs_stats.enqueue_err_count;
+	stats->dequeue_err_count = bcmfs_stats.dequeue_err_count;
+}
+
+static void
+bcmfs_sym_stats_reset(struct rte_cryptodev *dev)
+{
+	struct bcmfs_sym_dev_private *bcmfs_priv;
+	struct bcmfs_device *fsdev;
+
+	if (dev == NULL) {
+		BCMFS_LOG(ERR, "invalid cryptodev ptr %p", dev);
+		return;
+	}
+	bcmfs_priv = dev->data->dev_private;
+	fsdev = bcmfs_priv->fsdev;
+
+	bcmfs_qp_stats_reset(fsdev->qps_in_use, fsdev->max_hw_qps);
+}
+
+static int
+bcmfs_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
+{
+	struct bcmfs_sym_dev_private *bcmfs_private = dev->data->dev_private;
+	struct bcmfs_qp *qp = (struct bcmfs_qp *)
+			      (dev->data->queue_pairs[queue_pair_id]);
+
+	BCMFS_LOG(DEBUG, "Release sym qp %u on device %d",
+		  queue_pair_id, dev->data->dev_id);
+
+	rte_mempool_free(qp->sr_mp);
+
+	bcmfs_private->fsdev->qps_in_use[queue_pair_id] = NULL;
+
+	return bcmfs_qp_release((struct bcmfs_qp **)
+				&dev->data->queue_pairs[queue_pair_id]);
+}
+
+static void
+spu_req_init(struct bcmfs_sym_request *sr, rte_iova_t iova __rte_unused)
+{
+	memset(sr, 0, sizeof(*sr));
+}
+
+static void
+req_pool_obj_init(__rte_unused struct rte_mempool *mp,
+		  __rte_unused void *opaque, void *obj,
+		  __rte_unused unsigned int obj_idx)
+{
+	spu_req_init(obj, rte_mempool_virt2iova(obj));
+}
+
+static struct rte_mempool *
+bcmfs_sym_req_pool_create(struct rte_cryptodev *cdev __rte_unused,
+			  uint32_t nobjs, uint16_t qp_id,
+			  int socket_id)
+{
+	char softreq_pool_name[RTE_RING_NAMESIZE];
+	struct rte_mempool *mp;
+
+	snprintf(softreq_pool_name, RTE_RING_NAMESIZE, "%s_%d",
+		 "bcm_sym", qp_id);
+
+	mp = rte_mempool_create(softreq_pool_name,
+				RTE_ALIGN_MUL_CEIL(nobjs, 64),
+				sizeof(struct bcmfs_sym_request),
+				64, 0, NULL, NULL, req_pool_obj_init, NULL,
+				socket_id, 0);
+	if (mp == NULL)
+		BCMFS_LOG(ERR, "Failed to create req pool, qid %d, err %d",
+				qp_id, rte_errno);
+
+	return mp;
+}
+
+static int
+bcmfs_sym_qp_setup(struct rte_cryptodev *cdev, uint16_t qp_id,
+		   const struct rte_cryptodev_qp_conf *qp_conf,
+		   int socket_id)
+{
+	int ret = 0;
+	struct bcmfs_qp *qp = NULL;
+	struct bcmfs_qp_config bcmfs_qp_conf;
+
+	struct bcmfs_qp **qp_addr =
+			(struct bcmfs_qp **)&cdev->data->queue_pairs[qp_id];
+	struct bcmfs_sym_dev_private *bcmfs_private = cdev->data->dev_private;
+	struct bcmfs_device *fsdev = bcmfs_private->fsdev;
+
+
+	/* If qp is already in use free ring memory and qp metadata. */
+	if (*qp_addr != NULL) {
+		ret = bcmfs_sym_qp_release(cdev, qp_id);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (qp_id >= fsdev->max_hw_qps) {
+		BCMFS_LOG(ERR, "qp_id %u invalid for this device", qp_id);
+		return -EINVAL;
+	}
+
+	bcmfs_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
+	bcmfs_qp_conf.socket_id = socket_id;
+	bcmfs_qp_conf.max_descs_req = BCMFS_CRYPTO_MAX_HW_DESCS_PER_REQ;
+	bcmfs_qp_conf.iobase = BCMFS_QP_IOBASE_XLATE(fsdev->mmap_addr, qp_id);
+	bcmfs_qp_conf.ops = fsdev->sym_hw_qp_ops;
+
+	ret = bcmfs_qp_setup(qp_addr, qp_id, &bcmfs_qp_conf);
+	if (ret != 0)
+		return ret;
+
+	qp = (struct bcmfs_qp *)*qp_addr;
+
+	qp->sr_mp = bcmfs_sym_req_pool_create(cdev, qp_conf->nb_descriptors,
+					      qp_id, socket_id);
+	if (qp->sr_mp == NULL)
+		return -ENOMEM;
+
+	/* store a link to the qp in the bcmfs_device */
+	bcmfs_private->fsdev->qps_in_use[qp_id] = *qp_addr;
+
+	cdev->data->queue_pairs[qp_id] = qp;
+	BCMFS_LOG(NOTICE, "queue %d setup done\n", qp_id);
+
+	return 0;
+}
+
+static struct rte_cryptodev_ops crypto_bcmfs_ops = {
+	/* Device related operations */
+	.dev_configure          = bcmfs_sym_dev_config,
+	.dev_start              = bcmfs_sym_dev_start,
+	.dev_stop               = bcmfs_sym_dev_stop,
+	.dev_close              = bcmfs_sym_dev_close,
+	.dev_infos_get          = bcmfs_sym_dev_info_get,
+	/* Stats Collection */
+	.stats_get              = bcmfs_sym_stats_get,
+	.stats_reset            = bcmfs_sym_stats_reset,
+	/* Queue-Pair management */
+	.queue_pair_setup       = bcmfs_sym_qp_setup,
+	.queue_pair_release     = bcmfs_sym_qp_release,
+};
+
+/** Enqueue burst */
+static uint16_t
+bcmfs_sym_pmd_enqueue_op_burst(void *queue_pair,
+			       struct rte_crypto_op **ops,
+			       uint16_t nb_ops)
+{
+	int i, j;
+	uint16_t enq = 0;
+	struct bcmfs_sym_request *sreq;
+	struct bcmfs_qp *qp = (struct bcmfs_qp *)queue_pair;
+
+	if (nb_ops == 0)
+		return 0;
+
+	if (nb_ops > BCMFS_MAX_REQS_BUFF)
+		nb_ops = BCMFS_MAX_REQS_BUFF;
+
+	 /* We do not process more than available space */
+	if (nb_ops >  (qp->nb_descriptors - qp->nb_pending_requests))
+		nb_ops = qp->nb_descriptors - qp->nb_pending_requests;
+
+	for (i = 0; i < nb_ops; i++) {
+		if (rte_mempool_get(qp->sr_mp, (void **)&sreq))
+			goto enqueue_err;
+
+		/* save rte_crypto_op */
+		sreq->op = ops[i];
+
+		/* save context */
+		qp->infl_msgs[i] = &sreq->msgs;
+		qp->infl_msgs[i]->ctx = (void *)sreq;
+	}
+	/* Send burst request to hw QP */
+	enq = bcmfs_enqueue_op_burst(qp, (void **)qp->infl_msgs, i);
+
+	for (j = enq; j < i; j++)
+		rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx);
+
+	return enq;
+
+enqueue_err:
+	for (j = 0; j < i; j++)
+		rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx);
+
+	return enq;
+}
+
+static uint16_t
+bcmfs_sym_pmd_dequeue_op_burst(void *queue_pair,
+			       struct rte_crypto_op **ops,
+			       uint16_t nb_ops)
+{
+	int i;
+	uint16_t deq = 0;
+	unsigned int pkts = 0;
+	struct bcmfs_sym_request *sreq;
+	struct bcmfs_qp *qp = queue_pair;
+
+	if (nb_ops > BCMFS_MAX_REQS_BUFF)
+		nb_ops = BCMFS_MAX_REQS_BUFF;
+
+	deq = bcmfs_dequeue_op_burst(qp, (void **)qp->infl_msgs, nb_ops);
+	/* get rte_crypto_ops */
+	for (i = 0; i < deq; i++) {
+		sreq = (struct bcmfs_sym_request *)qp->infl_msgs[i]->ctx;
+
+		ops[pkts++] = sreq->op;
+
+		rte_mempool_put(qp->sr_mp, sreq);
+	}
+
+	return pkts;
+}
+
+/*
+ * An rte_driver is needed in the registration of both the
+ * device and the driver with cryptodev.
+ */
+static const char bcmfs_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_BCMFS_SYM_PMD);
+static const struct rte_driver cryptodev_bcmfs_sym_driver = {
+	.name = bcmfs_sym_drv_name,
+	.alias = bcmfs_sym_drv_name
+};
+
+int
+bcmfs_sym_dev_create(struct bcmfs_device *fsdev)
+{
+	struct rte_cryptodev_pmd_init_params init_params = {
+		.name = "",
+		.socket_id = rte_socket_id(),
+		.private_data_size = sizeof(struct bcmfs_sym_dev_private)
+	};
+	char name[RTE_CRYPTODEV_NAME_MAX_LEN];
+	struct rte_cryptodev *cryptodev;
+	struct bcmfs_sym_dev_private *internals;
+
+	snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
+		 fsdev->name, "sym");
+
+	/* Populate subset device to use in cryptodev device creation */
+	fsdev->sym_rte_dev.driver = &cryptodev_bcmfs_sym_driver;
+	fsdev->sym_rte_dev.numa_node = 0;
+	fsdev->sym_rte_dev.devargs = NULL;
+
+	cryptodev = rte_cryptodev_pmd_create(name,
+					     &fsdev->sym_rte_dev,
+					     &init_params);
+	if (cryptodev == NULL)
+		return -ENODEV;
+
+	fsdev->sym_rte_dev.name = cryptodev->data->name;
+	cryptodev->driver_id = cryptodev_bcmfs_driver_id;
+	cryptodev->dev_ops = &crypto_bcmfs_ops;
+
+	cryptodev->enqueue_burst = bcmfs_sym_pmd_enqueue_op_burst;
+	cryptodev->dequeue_burst = bcmfs_sym_pmd_dequeue_op_burst;
+
+	cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
+				   RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
+				   RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
+
+	internals = cryptodev->data->dev_private;
+	internals->fsdev = fsdev;
+	fsdev->sym_dev = internals;
+
+	internals->sym_dev_id = cryptodev->data->dev_id;
+
+	BCMFS_LOG(DEBUG, "Created bcmfs-sym device %s as cryptodev instance %d",
+		  cryptodev->data->name, internals->sym_dev_id);
+	return 0;
+}
+
+int
+bcmfs_sym_dev_destroy(struct bcmfs_device *fsdev)
+{
+	struct rte_cryptodev *cryptodev;
+
+	if (fsdev == NULL)
+		return -ENODEV;
+	if (fsdev->sym_dev == NULL)
+		return 0;
+
+	/* free crypto device */
+	cryptodev = rte_cryptodev_pmd_get_dev(fsdev->sym_dev->sym_dev_id);
+	rte_cryptodev_pmd_destroy(cryptodev);
+	fsdev->sym_rte_dev.name = NULL;
+	fsdev->sym_dev = NULL;
+
+	return 0;
+}
+
+static struct cryptodev_driver bcmfs_crypto_drv;
+RTE_PMD_REGISTER_CRYPTO_DRIVER(bcmfs_crypto_drv,
+			       cryptodev_bcmfs_sym_driver,
+			       cryptodev_bcmfs_driver_id);

drivers/crypto/bcmfs/bcmfs_sym_pmd.h

@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Broadcom
+ * All rights reserved.
+ */
+
+#ifndef _BCMFS_SYM_PMD_H_
+#define _BCMFS_SYM_PMD_H_
+
+#include <rte_cryptodev.h>
+
+#include "bcmfs_device.h"
+
+#define CRYPTODEV_NAME_BCMFS_SYM_PMD	crypto_bcmfs
+
+#define BCMFS_CRYPTO_MAX_HW_DESCS_PER_REQ	16
+
+extern uint8_t cryptodev_bcmfs_driver_id;
+
+/** private data structure for a BCMFS device.
+ *  This BCMFS device is a device offering only symmetric crypto service,
+ *  there can be one of these on each bcmfs_pci_device (VF).
+ */
+struct bcmfs_sym_dev_private {
+	/* The bcmfs device hosting the service */
+	struct bcmfs_device *fsdev;
+	/* Device instance for this rte_cryptodev */
+	uint8_t sym_dev_id;
+	/* BCMFS device symmetric crypto capabilities */
+	const struct rte_cryptodev_capabilities *fsdev_capabilities;
+};
+
+int
+bcmfs_sym_dev_create(struct bcmfs_device *fdev);
+
+int
+bcmfs_sym_dev_destroy(struct bcmfs_device *fdev);
+
+#endif /* _BCMFS_SYM_PMD_H_ */

drivers/crypto/bcmfs/bcmfs_sym_req.h

@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2020 Broadcom
+ * All rights reserved.
+ */
+
+#ifndef _BCMFS_SYM_REQ_H_
+#define _BCMFS_SYM_REQ_H_
+
+#include "bcmfs_dev_msg.h"
+
+/*
+ * This structure hold the supportive data required to process a
+ * rte_crypto_op
+ */
+struct bcmfs_sym_request {
+	/* bcmfs qp message for h/w queues to process */
+	struct bcmfs_qp_message msgs;
+	/* crypto op */
+	struct rte_crypto_op *op;
+};
+
+#endif /* _BCMFS_SYM_REQ_H_ */

drivers/crypto/bcmfs/meson.build

@@ -11,5 +11,6 @@ sources = files(
 		'bcmfs_qp.c',
 		'hw/bcmfs4_rm.c',
 		'hw/bcmfs5_rm.c',
-		'hw/bcmfs_rm_common.c'
+		'hw/bcmfs_rm_common.c',
+		'bcmfs_sym_pmd.c'
 		)