numam-dpdk/drivers/crypto/virtio/virtio_cryptodev.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
 */
#include <stdbool.h>
#include <unistd.h>

#include <rte_common.h>
#include <rte_errno.h>
#include <rte_pci.h>
#include <rte_bus_pci.h>
#include <rte_cryptodev.h>
#include <rte_cryptodev_pmd.h>
#include <rte_eal.h>

#include "virtio_cryptodev.h"
#include "virtqueue.h"
#include "virtio_crypto_algs.h"
#include "virtio_crypto_capabilities.h"

static int virtio_crypto_dev_configure(struct rte_cryptodev *dev,
		struct rte_cryptodev_config *config);
static int virtio_crypto_dev_start(struct rte_cryptodev *dev);
static void virtio_crypto_dev_stop(struct rte_cryptodev *dev);
static int virtio_crypto_dev_close(struct rte_cryptodev *dev);
static void virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
		struct rte_cryptodev_info *dev_info);
static void virtio_crypto_dev_stats_get(struct rte_cryptodev *dev,
		struct rte_cryptodev_stats *stats);
static void virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev);
static int virtio_crypto_qp_setup(struct rte_cryptodev *dev,
		uint16_t queue_pair_id,
		const struct rte_cryptodev_qp_conf *qp_conf,
		int socket_id);
static int virtio_crypto_qp_release(struct rte_cryptodev *dev,
		uint16_t queue_pair_id);
static void virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev);
static unsigned int virtio_crypto_sym_get_session_private_size(
		struct rte_cryptodev *dev);
static void virtio_crypto_sym_clear_session(struct rte_cryptodev *dev,
		struct rte_cryptodev_sym_session *sess);
static int virtio_crypto_sym_configure_session(struct rte_cryptodev *dev,
		struct rte_crypto_sym_xform *xform,
		struct rte_cryptodev_sym_session *session,
		struct rte_mempool *mp);

/*
 * The set of PCI devices this driver supports
 */
static const struct rte_pci_id pci_id_virtio_crypto_map[] = {
	{ RTE_PCI_DEVICE(VIRTIO_CRYPTO_PCI_VENDORID,
				VIRTIO_CRYPTO_PCI_DEVICEID) },
	{ .vendor_id = 0, /* sentinel */ },
};

static const struct rte_cryptodev_capabilities virtio_capabilities[] = {
	VIRTIO_SYM_CAPABILITIES,
	RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
};

uint8_t cryptodev_virtio_driver_id;

#define NUM_ENTRY_SYM_CREATE_SESSION 4

static int
virtio_crypto_send_command(struct virtqueue *vq,
		struct virtio_crypto_op_ctrl_req *ctrl, uint8_t *cipher_key,
		uint8_t *auth_key, struct virtio_crypto_session *session)
{
	uint8_t idx = 0;
	uint8_t needed = 1;
	uint32_t head = 0;
	uint32_t len_cipher_key = 0;
	uint32_t len_auth_key = 0;
	uint32_t len_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req);
	uint32_t len_session_input = sizeof(struct virtio_crypto_session_input);
	uint32_t len_total = 0;
	uint32_t input_offset = 0;
	void *virt_addr_started = NULL;
	phys_addr_t phys_addr_started;
	struct vring_desc *desc;
	uint32_t desc_offset;
	struct virtio_crypto_session_input *input;
	int ret;

	PMD_INIT_FUNC_TRACE();

	if (session == NULL) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("session is NULL.");
		return -EINVAL;
	}
	/* cipher only is supported, it is available if auth_key is NULL */
	if (!cipher_key) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("cipher key is NULL.");
		return -EINVAL;
	}

	head = vq->vq_desc_head_idx;
	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_desc_head_idx = %d, vq = %p",
					head, vq);

	if (vq->vq_free_cnt < needed) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("Not enough entry");
		return -ENOSPC;
	}

	/* calculate the length of cipher key */
	if (cipher_key) {
		switch (ctrl->u.sym_create_session.op_type) {
		case VIRTIO_CRYPTO_SYM_OP_CIPHER:
			len_cipher_key
				= ctrl->u.sym_create_session.u.cipher
							.para.keylen;
			break;
		case VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING:
			len_cipher_key
				= ctrl->u.sym_create_session.u.chain
					.para.cipher_param.keylen;
			break;
		default:
			VIRTIO_CRYPTO_SESSION_LOG_ERR("invalid op type");
			return -EINVAL;
		}
	}

	/* calculate the length of auth key */
	if (auth_key) {
		len_auth_key =
			ctrl->u.sym_create_session.u.chain.para.u.mac_param
				.auth_key_len;
	}

	/*
	 * malloc memory to store indirect vring_desc entries, including
	 * ctrl request, cipher key, auth key, session input and desc vring
	 */
	desc_offset = len_ctrl_req + len_cipher_key + len_auth_key
		+ len_session_input;
	virt_addr_started = rte_malloc(NULL,
		desc_offset + NUM_ENTRY_SYM_CREATE_SESSION
			* sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE);
	if (virt_addr_started == NULL) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap memory");
		return -ENOSPC;
	}
	phys_addr_started = rte_malloc_virt2iova(virt_addr_started);

	/* address to store indirect vring desc entries */
	desc = (struct vring_desc *)
		((uint8_t *)virt_addr_started + desc_offset);

	/*  ctrl req part */
	memcpy(virt_addr_started, ctrl, len_ctrl_req);
	desc[idx].addr = phys_addr_started;
	desc[idx].len = len_ctrl_req;
	desc[idx].flags = VRING_DESC_F_NEXT;
	desc[idx].next = idx + 1;
	idx++;
	len_total += len_ctrl_req;
	input_offset += len_ctrl_req;

	/* cipher key part */
	if (len_cipher_key > 0) {
		memcpy((uint8_t *)virt_addr_started + len_total,
			cipher_key, len_cipher_key);

		desc[idx].addr = phys_addr_started + len_total;
		desc[idx].len = len_cipher_key;
		desc[idx].flags = VRING_DESC_F_NEXT;
		desc[idx].next = idx + 1;
		idx++;
		len_total += len_cipher_key;
		input_offset += len_cipher_key;
	}

	/* auth key part */
	if (len_auth_key > 0) {
		memcpy((uint8_t *)virt_addr_started + len_total,
			auth_key, len_auth_key);

		desc[idx].addr = phys_addr_started + len_total;
		desc[idx].len = len_auth_key;
		desc[idx].flags = VRING_DESC_F_NEXT;
		desc[idx].next = idx + 1;
		idx++;
		len_total += len_auth_key;
		input_offset += len_auth_key;
	}

	/* input part */
	input = (struct virtio_crypto_session_input *)
		((uint8_t *)virt_addr_started + input_offset);
	input->status = VIRTIO_CRYPTO_ERR;
	input->session_id = ~0ULL;
	desc[idx].addr = phys_addr_started + len_total;
	desc[idx].len = len_session_input;
	desc[idx].flags = VRING_DESC_F_WRITE;
	idx++;

	/* use a single desc entry */
	vq->vq_ring.desc[head].addr = phys_addr_started + desc_offset;
	vq->vq_ring.desc[head].len = idx * sizeof(struct vring_desc);
	vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT;
	vq->vq_free_cnt--;

	vq->vq_desc_head_idx = vq->vq_ring.desc[head].next;

	vq_update_avail_ring(vq, head);
	vq_update_avail_idx(vq);

	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d",
					vq->vq_queue_index);

	virtqueue_notify(vq);

	rte_rmb();
	while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) {
		rte_rmb();
		usleep(100);
	}

	while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) {
		uint32_t idx, desc_idx, used_idx;
		struct vring_used_elem *uep;

		used_idx = (uint32_t)(vq->vq_used_cons_idx
				& (vq->vq_nentries - 1));
		uep = &vq->vq_ring.used->ring[used_idx];
		idx = (uint32_t) uep->id;
		desc_idx = idx;

		while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) {
			desc_idx = vq->vq_ring.desc[desc_idx].next;
			vq->vq_free_cnt++;
		}

		vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
		vq->vq_desc_head_idx = idx;

		vq->vq_used_cons_idx++;
		vq->vq_free_cnt++;
	}

	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n"
			"vq->vq_desc_head_idx=%d",
			vq->vq_free_cnt, vq->vq_desc_head_idx);

	/* get the result */
	if (input->status != VIRTIO_CRYPTO_OK) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("Something wrong on backend! "
				"status=%u, session_id=%" PRIu64 "",
				input->status, input->session_id);
		rte_free(virt_addr_started);
		ret = -1;
	} else {
		session->session_id = input->session_id;

		VIRTIO_CRYPTO_SESSION_LOG_INFO("Create session successfully, "
				"session_id=%" PRIu64 "", input->session_id);
		rte_free(virt_addr_started);
		ret = 0;
	}

	return ret;
}

void
virtio_crypto_queue_release(struct virtqueue *vq)
{
	struct virtio_crypto_hw *hw;

	PMD_INIT_FUNC_TRACE();

	if (vq) {
		hw = vq->hw;
		/* Select and deactivate the queue */
		VTPCI_OPS(hw)->del_queue(hw, vq);

		rte_memzone_free(vq->mz);
		rte_mempool_free(vq->mpool);
		rte_free(vq);
	}
}

#define MPOOL_MAX_NAME_SZ 32

int
virtio_crypto_queue_setup(struct rte_cryptodev *dev,
		int queue_type,
		uint16_t vtpci_queue_idx,
		uint16_t nb_desc,
		int socket_id,
		struct virtqueue **pvq)
{
	char vq_name[VIRTQUEUE_MAX_NAME_SZ];
	char mpool_name[MPOOL_MAX_NAME_SZ];
	const struct rte_memzone *mz;
	unsigned int vq_size, size;
	struct virtio_crypto_hw *hw = dev->data->dev_private;
	struct virtqueue *vq = NULL;
	uint32_t i = 0;
	uint32_t j;

	PMD_INIT_FUNC_TRACE();

	VIRTIO_CRYPTO_INIT_LOG_DBG("setting up queue: %u", vtpci_queue_idx);

	/*
	 * Read the virtqueue size from the Queue Size field
	 * Always power of 2 and if 0 virtqueue does not exist
	 */
	vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx);
	if (vq_size == 0) {
		VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue does not exist");
		return -EINVAL;
	}
	VIRTIO_CRYPTO_INIT_LOG_DBG("vq_size: %u", vq_size);

	if (!rte_is_power_of_2(vq_size)) {
		VIRTIO_CRYPTO_INIT_LOG_ERR("virtqueue size is not powerof 2");
		return -EINVAL;
	}

	if (queue_type == VTCRYPTO_DATAQ) {
		snprintf(vq_name, sizeof(vq_name), "dev%d_dataqueue%d",
				dev->data->dev_id, vtpci_queue_idx);
		snprintf(mpool_name, sizeof(mpool_name),
				"dev%d_dataqueue%d_mpool",
				dev->data->dev_id, vtpci_queue_idx);
	} else if (queue_type == VTCRYPTO_CTRLQ) {
		snprintf(vq_name, sizeof(vq_name), "dev%d_controlqueue",
				dev->data->dev_id);
		snprintf(mpool_name, sizeof(mpool_name),
				"dev%d_controlqueue_mpool",
				dev->data->dev_id);
	}
	size = RTE_ALIGN_CEIL(sizeof(*vq) +
				vq_size * sizeof(struct vq_desc_extra),
				RTE_CACHE_LINE_SIZE);
	vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
				socket_id);
	if (vq == NULL) {
		VIRTIO_CRYPTO_INIT_LOG_ERR("Can not allocate virtqueue");
		return -ENOMEM;
	}

	if (queue_type == VTCRYPTO_DATAQ) {
		/* pre-allocate a mempool and use it in the data plane to
		 * improve performance
		 */
		vq->mpool = rte_mempool_lookup(mpool_name);
		if (vq->mpool == NULL)
			vq->mpool = rte_mempool_create(mpool_name,
					vq_size,
					sizeof(struct virtio_crypto_op_cookie),
					RTE_CACHE_LINE_SIZE, 0,
					NULL, NULL, NULL, NULL, socket_id,
					0);
		if (!vq->mpool) {
			VIRTIO_CRYPTO_DRV_LOG_ERR("Virtio Crypto PMD "
					"Cannot create mempool");
			goto mpool_create_err;
		}
		for (i = 0; i < vq_size; i++) {
			vq->vq_descx[i].cookie =
				rte_zmalloc("crypto PMD op cookie pointer",
					sizeof(struct virtio_crypto_op_cookie),
					RTE_CACHE_LINE_SIZE);
			if (vq->vq_descx[i].cookie == NULL) {
				VIRTIO_CRYPTO_DRV_LOG_ERR("Failed to "
						"alloc mem for cookie");
				goto cookie_alloc_err;
			}
		}
	}

	vq->hw = hw;
	vq->dev_id = dev->data->dev_id;
	vq->vq_queue_index = vtpci_queue_idx;
	vq->vq_nentries = vq_size;

	/*
	 * Using part of the vring entries is permitted, but the maximum
	 * is vq_size
	 */
	if (nb_desc == 0 || nb_desc > vq_size)
		nb_desc = vq_size;
	vq->vq_free_cnt = nb_desc;

	/*
	 * Reserve a memzone for vring elements
	 */
	size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
	vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
	VIRTIO_CRYPTO_INIT_LOG_DBG("%s vring_size: %d, rounded_vring_size: %d",
			(queue_type == VTCRYPTO_DATAQ) ? "dataq" : "ctrlq",
			size, vq->vq_ring_size);

	mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
			socket_id, 0, VIRTIO_PCI_VRING_ALIGN);
	if (mz == NULL) {
		if (rte_errno == EEXIST)
			mz = rte_memzone_lookup(vq_name);
		if (mz == NULL) {
			VIRTIO_CRYPTO_INIT_LOG_ERR("not enough memory");
			goto mz_reserve_err;
		}
	}

	/*
	 * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit,
	 * and only accepts 32 bit page frame number.
	 * Check if the allocated physical memory exceeds 16TB.
	 */
	if ((mz->iova + vq->vq_ring_size - 1)
				>> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
		VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be "
					"above 16TB!");
		goto vring_addr_err;
	}

	memset(mz->addr, 0, sizeof(mz->len));
	vq->mz = mz;
	vq->vq_ring_mem = mz->iova;
	vq->vq_ring_virt_mem = mz->addr;
	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_mem(physical): 0x%"PRIx64,
					(uint64_t)mz->iova);
	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_ring_virt_mem: 0x%"PRIx64,
					(uint64_t)(uintptr_t)mz->addr);

	*pvq = vq;

	return 0;

vring_addr_err:
	rte_memzone_free(mz);
mz_reserve_err:
cookie_alloc_err:
	rte_mempool_free(vq->mpool);
	if (i != 0) {
		for (j = 0; j < i; j++)
			rte_free(vq->vq_descx[j].cookie);
	}
mpool_create_err:
	rte_free(vq);
	return -ENOMEM;
}

static int
virtio_crypto_ctrlq_setup(struct rte_cryptodev *dev, uint16_t queue_idx)
{
	int ret;
	struct virtqueue *vq;
	struct virtio_crypto_hw *hw = dev->data->dev_private;

	/* if virtio device has started, do not touch the virtqueues */
	if (dev->data->dev_started)
		return 0;

	PMD_INIT_FUNC_TRACE();

	ret = virtio_crypto_queue_setup(dev, VTCRYPTO_CTRLQ, queue_idx,
			0, SOCKET_ID_ANY, &vq);
	if (ret < 0) {
		VIRTIO_CRYPTO_INIT_LOG_ERR("control vq initialization failed");
		return ret;
	}

	hw->cvq = vq;

	return 0;
}

static void
virtio_crypto_free_queues(struct rte_cryptodev *dev)
{
	unsigned int i;
	struct virtio_crypto_hw *hw = dev->data->dev_private;

	PMD_INIT_FUNC_TRACE();

	/* control queue release */
	virtio_crypto_queue_release(hw->cvq);

	/* data queue release */
	for (i = 0; i < hw->max_dataqueues; i++)
		virtio_crypto_queue_release(dev->data->queue_pairs[i]);
}

static int
virtio_crypto_dev_close(struct rte_cryptodev *dev __rte_unused)
{
	return 0;
}

/*
 * dev_ops for virtio, bare necessities for basic operation
 */
static struct rte_cryptodev_ops virtio_crypto_dev_ops = {
	/* Device related operations */
	.dev_configure			 = virtio_crypto_dev_configure,
	.dev_start			 = virtio_crypto_dev_start,
	.dev_stop			 = virtio_crypto_dev_stop,
	.dev_close			 = virtio_crypto_dev_close,
	.dev_infos_get			 = virtio_crypto_dev_info_get,

	.stats_get			 = virtio_crypto_dev_stats_get,
	.stats_reset			 = virtio_crypto_dev_stats_reset,

	.queue_pair_setup                = virtio_crypto_qp_setup,
	.queue_pair_release              = virtio_crypto_qp_release,

	/* Crypto related operations */
	.sym_session_get_size		= virtio_crypto_sym_get_session_private_size,
	.sym_session_configure		= virtio_crypto_sym_configure_session,
	.sym_session_clear		= virtio_crypto_sym_clear_session
};

static void
virtio_crypto_update_stats(struct rte_cryptodev *dev,
		struct rte_cryptodev_stats *stats)
{
	unsigned int i;
	struct virtio_crypto_hw *hw = dev->data->dev_private;

	PMD_INIT_FUNC_TRACE();

	if (stats == NULL) {
		VIRTIO_CRYPTO_DRV_LOG_ERR("invalid pointer");
		return;
	}

	for (i = 0; i < hw->max_dataqueues; i++) {
		const struct virtqueue *data_queue
			= dev->data->queue_pairs[i];
		if (data_queue == NULL)
			continue;

		stats->enqueued_count += data_queue->packets_sent_total;
		stats->enqueue_err_count += data_queue->packets_sent_failed;

		stats->dequeued_count += data_queue->packets_received_total;
		stats->dequeue_err_count
			+= data_queue->packets_received_failed;
	}
}

static void
virtio_crypto_dev_stats_get(struct rte_cryptodev *dev,
		struct rte_cryptodev_stats *stats)
{
	PMD_INIT_FUNC_TRACE();

	virtio_crypto_update_stats(dev, stats);
}

static void
virtio_crypto_dev_stats_reset(struct rte_cryptodev *dev)
{
	unsigned int i;
	struct virtio_crypto_hw *hw = dev->data->dev_private;

	PMD_INIT_FUNC_TRACE();

	for (i = 0; i < hw->max_dataqueues; i++) {
		struct virtqueue *data_queue = dev->data->queue_pairs[i];
		if (data_queue == NULL)
			continue;

		data_queue->packets_sent_total = 0;
		data_queue->packets_sent_failed = 0;

		data_queue->packets_received_total = 0;
		data_queue->packets_received_failed = 0;
	}
}

static int
virtio_crypto_qp_setup(struct rte_cryptodev *dev, uint16_t queue_pair_id,
		const struct rte_cryptodev_qp_conf *qp_conf,
		int socket_id)
{
	int ret;
	struct virtqueue *vq;

	PMD_INIT_FUNC_TRACE();

	/* if virtio dev is started, do not touch the virtqueues */
	if (dev->data->dev_started)
		return 0;

	ret = virtio_crypto_queue_setup(dev, VTCRYPTO_DATAQ, queue_pair_id,
			qp_conf->nb_descriptors, socket_id, &vq);
	if (ret < 0) {
		VIRTIO_CRYPTO_INIT_LOG_ERR(
			"virtio crypto data queue initialization failed\n");
		return ret;
	}

	dev->data->queue_pairs[queue_pair_id] = vq;

	return 0;
}

static int
virtio_crypto_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
{
	struct virtqueue *vq
		= (struct virtqueue *)dev->data->queue_pairs[queue_pair_id];

	PMD_INIT_FUNC_TRACE();

	if (vq == NULL) {
		VIRTIO_CRYPTO_DRV_LOG_DBG("vq already freed");
		return 0;
	}

	virtio_crypto_queue_release(vq);
	return 0;
}

static int
virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features)
{
	uint64_t host_features;

	PMD_INIT_FUNC_TRACE();

	/* Prepare guest_features: feature that driver wants to support */
	VIRTIO_CRYPTO_INIT_LOG_DBG("guest_features before negotiate = %" PRIx64,
		req_features);

	/* Read device(host) feature bits */
	host_features = VTPCI_OPS(hw)->get_features(hw);
	VIRTIO_CRYPTO_INIT_LOG_DBG("host_features before negotiate = %" PRIx64,
		host_features);

	/*
	 * Negotiate features: Subset of device feature bits are written back
	 * guest feature bits.
	 */
	hw->guest_features = req_features;
	hw->guest_features = vtpci_cryptodev_negotiate_features(hw,
							host_features);
	VIRTIO_CRYPTO_INIT_LOG_DBG("features after negotiate = %" PRIx64,
		hw->guest_features);

	if (hw->modern) {
		if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) {
			VIRTIO_CRYPTO_INIT_LOG_ERR(
				"VIRTIO_F_VERSION_1 features is not enabled.");
			return -1;
		}
		vtpci_cryptodev_set_status(hw,
			VIRTIO_CONFIG_STATUS_FEATURES_OK);
		if (!(vtpci_cryptodev_get_status(hw) &
			VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
			VIRTIO_CRYPTO_INIT_LOG_ERR("failed to set FEATURES_OK "
						"status!");
			return -1;
		}
	}

	hw->req_guest_features = req_features;

	return 0;
}

/* reset device and renegotiate features if needed */
static int
virtio_crypto_init_device(struct rte_cryptodev *cryptodev,
	uint64_t req_features)
{
	struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
	struct virtio_crypto_config local_config;
	struct virtio_crypto_config *config = &local_config;

	PMD_INIT_FUNC_TRACE();

	/* Reset the device although not necessary at startup */
	vtpci_cryptodev_reset(hw);

	/* Tell the host we've noticed this device. */
	vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);

	/* Tell the host we've known how to drive the device. */
	vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
	if (virtio_negotiate_features(hw, req_features) < 0)
		return -1;

	/* Get status of the device */
	vtpci_read_cryptodev_config(hw,
		offsetof(struct virtio_crypto_config, status),
		&config->status, sizeof(config->status));
	if (config->status != VIRTIO_CRYPTO_S_HW_READY) {
		VIRTIO_CRYPTO_DRV_LOG_ERR("accelerator hardware is "
				"not ready");
		return -1;
	}

	/* Get number of data queues */
	vtpci_read_cryptodev_config(hw,
		offsetof(struct virtio_crypto_config, max_dataqueues),
		&config->max_dataqueues,
		sizeof(config->max_dataqueues));
	hw->max_dataqueues = config->max_dataqueues;

	VIRTIO_CRYPTO_INIT_LOG_DBG("hw->max_dataqueues=%d",
		hw->max_dataqueues);

	return 0;
}

/*
 * This function is based on probe() function
 * It returns 0 on success.
 */
static int
crypto_virtio_create(const char *name, struct rte_pci_device *pci_dev,
		struct rte_cryptodev_pmd_init_params *init_params)
{
	struct rte_cryptodev *cryptodev;
	struct virtio_crypto_hw *hw;

	PMD_INIT_FUNC_TRACE();

	cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device,
					init_params);
	if (cryptodev == NULL)
		return -ENODEV;

	cryptodev->driver_id = cryptodev_virtio_driver_id;
	cryptodev->dev_ops = &virtio_crypto_dev_ops;

	cryptodev->enqueue_burst = virtio_crypto_pkt_tx_burst;
	cryptodev->dequeue_burst = virtio_crypto_pkt_rx_burst;

	cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
		RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
		RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;

	hw = cryptodev->data->dev_private;
	hw->dev_id = cryptodev->data->dev_id;
	hw->virtio_dev_capabilities = virtio_capabilities;

	VIRTIO_CRYPTO_INIT_LOG_DBG("dev %d vendorID=0x%x deviceID=0x%x",
		cryptodev->data->dev_id, pci_dev->id.vendor_id,
		pci_dev->id.device_id);

	/* pci device init */
	if (vtpci_cryptodev_init(pci_dev, hw))
		return -1;

	if (virtio_crypto_init_device(cryptodev,
			VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
		return -1;

	return 0;
}

static int
virtio_crypto_dev_uninit(struct rte_cryptodev *cryptodev)
{
	struct virtio_crypto_hw *hw = cryptodev->data->dev_private;

	PMD_INIT_FUNC_TRACE();

	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
		return -EPERM;

	if (cryptodev->data->dev_started) {
		virtio_crypto_dev_stop(cryptodev);
		virtio_crypto_dev_close(cryptodev);
	}

	cryptodev->dev_ops = NULL;
	cryptodev->enqueue_burst = NULL;
	cryptodev->dequeue_burst = NULL;

	/* release control queue */
	virtio_crypto_queue_release(hw->cvq);

	rte_free(cryptodev->data);
	cryptodev->data = NULL;

	VIRTIO_CRYPTO_DRV_LOG_INFO("dev_uninit completed");

	return 0;
}

static int
virtio_crypto_dev_configure(struct rte_cryptodev *cryptodev,
	struct rte_cryptodev_config *config __rte_unused)
{
	struct virtio_crypto_hw *hw = cryptodev->data->dev_private;

	PMD_INIT_FUNC_TRACE();

	if (virtio_crypto_init_device(cryptodev,
			VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
		return -1;

	/* setup control queue
	 * [0, 1, ... ,(config->max_dataqueues - 1)] are data queues
	 * config->max_dataqueues is the control queue
	 */
	if (virtio_crypto_ctrlq_setup(cryptodev, hw->max_dataqueues) < 0) {
		VIRTIO_CRYPTO_INIT_LOG_ERR("control queue setup error");
		return -1;
	}
	virtio_crypto_ctrlq_start(cryptodev);

	return 0;
}

static void
virtio_crypto_dev_stop(struct rte_cryptodev *dev)
{
	struct virtio_crypto_hw *hw = dev->data->dev_private;

	PMD_INIT_FUNC_TRACE();
	VIRTIO_CRYPTO_DRV_LOG_DBG("virtio_dev_stop");

	vtpci_cryptodev_reset(hw);

	virtio_crypto_dev_free_mbufs(dev);
	virtio_crypto_free_queues(dev);

	dev->data->dev_started = 0;
}

static int
virtio_crypto_dev_start(struct rte_cryptodev *dev)
{
	struct virtio_crypto_hw *hw = dev->data->dev_private;

	if (dev->data->dev_started)
		return 0;

	/* Do final configuration before queue engine starts */
	virtio_crypto_dataq_start(dev);
	vtpci_cryptodev_reinit_complete(hw);

	dev->data->dev_started = 1;

	return 0;
}

static void
virtio_crypto_dev_free_mbufs(struct rte_cryptodev *dev)
{
	uint32_t i;
	struct virtio_crypto_hw *hw = dev->data->dev_private;

	for (i = 0; i < hw->max_dataqueues; i++) {
		VIRTIO_CRYPTO_INIT_LOG_DBG("Before freeing dataq[%d] used "
			"and unused buf", i);
		VIRTQUEUE_DUMP((struct virtqueue *)
			dev->data->queue_pairs[i]);

		VIRTIO_CRYPTO_INIT_LOG_DBG("queue_pairs[%d]=%p",
				i, dev->data->queue_pairs[i]);

		virtqueue_detatch_unused(dev->data->queue_pairs[i]);

		VIRTIO_CRYPTO_INIT_LOG_DBG("After freeing dataq[%d] used and "
					"unused buf", i);
		VIRTQUEUE_DUMP(
			(struct virtqueue *)dev->data->queue_pairs[i]);
	}
}

static unsigned int
virtio_crypto_sym_get_session_private_size(
		struct rte_cryptodev *dev __rte_unused)
{
	PMD_INIT_FUNC_TRACE();

	return RTE_ALIGN_CEIL(sizeof(struct virtio_crypto_session), 16);
}

static int
virtio_crypto_check_sym_session_paras(
		struct rte_cryptodev *dev)
{
	struct virtio_crypto_hw *hw;

	PMD_INIT_FUNC_TRACE();

	if (unlikely(dev == NULL)) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("dev is NULL");
		return -1;
	}
	if (unlikely(dev->data == NULL)) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("dev->data is NULL");
		return -1;
	}
	hw = dev->data->dev_private;
	if (unlikely(hw == NULL)) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("hw is NULL");
		return -1;
	}
	if (unlikely(hw->cvq == NULL)) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("vq is NULL");
		return -1;
	}

	return 0;
}

static int
virtio_crypto_check_sym_clear_session_paras(
		struct rte_cryptodev *dev,
		struct rte_cryptodev_sym_session *sess)
{
	PMD_INIT_FUNC_TRACE();

	if (sess == NULL) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("sym_session is NULL");
		return -1;
	}

	return virtio_crypto_check_sym_session_paras(dev);
}

#define NUM_ENTRY_SYM_CLEAR_SESSION 2

static void
virtio_crypto_sym_clear_session(
		struct rte_cryptodev *dev,
		struct rte_cryptodev_sym_session *sess)
{
	struct virtio_crypto_hw *hw;
	struct virtqueue *vq;
	struct virtio_crypto_session *session;
	struct virtio_crypto_op_ctrl_req *ctrl;
	struct vring_desc *desc;
	uint8_t *status;
	uint8_t needed = 1;
	uint32_t head;
	uint8_t *malloc_virt_addr;
	uint64_t malloc_phys_addr;
	uint8_t len_inhdr = sizeof(struct virtio_crypto_inhdr);
	uint32_t len_op_ctrl_req = sizeof(struct virtio_crypto_op_ctrl_req);
	uint32_t desc_offset = len_op_ctrl_req + len_inhdr;

	PMD_INIT_FUNC_TRACE();

	if (virtio_crypto_check_sym_clear_session_paras(dev, sess) < 0)
		return;

	hw = dev->data->dev_private;
	vq = hw->cvq;
	session = (struct virtio_crypto_session *)get_sym_session_private_data(
		sess, cryptodev_virtio_driver_id);
	if (session == NULL) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid session parameter");
		return;
	}

	VIRTIO_CRYPTO_SESSION_LOG_INFO("vq->vq_desc_head_idx = %d, "
			"vq = %p", vq->vq_desc_head_idx, vq);

	if (vq->vq_free_cnt < needed) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR(
				"vq->vq_free_cnt = %d is less than %d, "
				"not enough", vq->vq_free_cnt, needed);
		return;
	}

	/*
	 * malloc memory to store information of ctrl request op,
	 * returned status and desc vring
	 */
	malloc_virt_addr = rte_malloc(NULL, len_op_ctrl_req + len_inhdr
		+ NUM_ENTRY_SYM_CLEAR_SESSION
		* sizeof(struct vring_desc), RTE_CACHE_LINE_SIZE);
	if (malloc_virt_addr == NULL) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("not enough heap room");
		return;
	}
	malloc_phys_addr = rte_malloc_virt2iova(malloc_virt_addr);

	/* assign ctrl request op part */
	ctrl = (struct virtio_crypto_op_ctrl_req *)malloc_virt_addr;
	ctrl->header.opcode = VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION;
	/* default data virtqueue is 0 */
	ctrl->header.queue_id = 0;
	ctrl->u.destroy_session.session_id = session->session_id;

	/* status part */
	status = &(((struct virtio_crypto_inhdr *)
		((uint8_t *)malloc_virt_addr + len_op_ctrl_req))->status);
	*status = VIRTIO_CRYPTO_ERR;

	/* indirect desc vring part */
	desc = (struct vring_desc *)((uint8_t *)malloc_virt_addr
		+ desc_offset);

	/* ctrl request part */
	desc[0].addr = malloc_phys_addr;
	desc[0].len = len_op_ctrl_req;
	desc[0].flags = VRING_DESC_F_NEXT;
	desc[0].next = 1;

	/* status part */
	desc[1].addr = malloc_phys_addr + len_op_ctrl_req;
	desc[1].len = len_inhdr;
	desc[1].flags = VRING_DESC_F_WRITE;

	/* use only a single desc entry */
	head = vq->vq_desc_head_idx;
	vq->vq_ring.desc[head].flags = VRING_DESC_F_INDIRECT;
	vq->vq_ring.desc[head].addr = malloc_phys_addr + desc_offset;
	vq->vq_ring.desc[head].len
		= NUM_ENTRY_SYM_CLEAR_SESSION
		* sizeof(struct vring_desc);
	vq->vq_free_cnt -= needed;

	vq->vq_desc_head_idx = vq->vq_ring.desc[head].next;

	vq_update_avail_ring(vq, head);
	vq_update_avail_idx(vq);

	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_queue_index = %d",
					vq->vq_queue_index);

	virtqueue_notify(vq);

	rte_rmb();
	while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) {
		rte_rmb();
		usleep(100);
	}

	while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) {
		uint32_t idx, desc_idx, used_idx;
		struct vring_used_elem *uep;

		used_idx = (uint32_t)(vq->vq_used_cons_idx
				& (vq->vq_nentries - 1));
		uep = &vq->vq_ring.used->ring[used_idx];
		idx = (uint32_t) uep->id;
		desc_idx = idx;
		while (vq->vq_ring.desc[desc_idx].flags
				& VRING_DESC_F_NEXT) {
			desc_idx = vq->vq_ring.desc[desc_idx].next;
			vq->vq_free_cnt++;
		}

		vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
		vq->vq_desc_head_idx = idx;
		vq->vq_used_cons_idx++;
		vq->vq_free_cnt++;
	}

	if (*status != VIRTIO_CRYPTO_OK) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("Close session failed "
				"status=%"PRIu32", session_id=%"PRIu64"",
				*status, session->session_id);
		rte_free(malloc_virt_addr);
		return;
	}

	VIRTIO_CRYPTO_INIT_LOG_DBG("vq->vq_free_cnt=%d\n"
			"vq->vq_desc_head_idx=%d",
			vq->vq_free_cnt, vq->vq_desc_head_idx);

	VIRTIO_CRYPTO_SESSION_LOG_INFO("Close session %"PRIu64" successfully ",
			session->session_id);

	memset(session, 0, sizeof(struct virtio_crypto_session));
	struct rte_mempool *sess_mp = rte_mempool_from_obj(session);
	set_sym_session_private_data(sess, cryptodev_virtio_driver_id, NULL);
	rte_mempool_put(sess_mp, session);
	rte_free(malloc_virt_addr);
}

static struct rte_crypto_cipher_xform *
virtio_crypto_get_cipher_xform(struct rte_crypto_sym_xform *xform)
{
	do {
		if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
			return &xform->cipher;

		xform = xform->next;
	} while (xform);

	return NULL;
}

static struct rte_crypto_auth_xform *
virtio_crypto_get_auth_xform(struct rte_crypto_sym_xform *xform)
{
	do {
		if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
			return &xform->auth;

		xform = xform->next;
	} while (xform);

	return NULL;
}

/** Get xform chain order */
static int
virtio_crypto_get_chain_order(struct rte_crypto_sym_xform *xform)
{
	if (xform == NULL)
		return -1;

	/* Cipher Only */
	if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
			xform->next == NULL)
		return VIRTIO_CRYPTO_CMD_CIPHER;

	/* Authentication Only */
	if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
			xform->next == NULL)
		return VIRTIO_CRYPTO_CMD_AUTH;

	/* Authenticate then Cipher */
	if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
			xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
		return VIRTIO_CRYPTO_CMD_HASH_CIPHER;

	/* Cipher then Authenticate */
	if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
			xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
		return VIRTIO_CRYPTO_CMD_CIPHER_HASH;

	return -1;
}

static int
virtio_crypto_sym_pad_cipher_param(
		struct virtio_crypto_cipher_session_para *para,
		struct rte_crypto_cipher_xform *cipher_xform)
{
	switch (cipher_xform->algo) {
	case RTE_CRYPTO_CIPHER_AES_CBC:
		para->algo = VIRTIO_CRYPTO_CIPHER_AES_CBC;
		break;
	default:
		VIRTIO_CRYPTO_SESSION_LOG_ERR("Crypto: Unsupported "
				"Cipher alg %u", cipher_xform->algo);
		return -1;
	}

	para->keylen = cipher_xform->key.length;
	switch (cipher_xform->op) {
	case RTE_CRYPTO_CIPHER_OP_ENCRYPT:
		para->op = VIRTIO_CRYPTO_OP_ENCRYPT;
		break;
	case RTE_CRYPTO_CIPHER_OP_DECRYPT:
		para->op = VIRTIO_CRYPTO_OP_DECRYPT;
		break;
	default:
		VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported cipher operation "
					"parameter");
		return -1;
	}

	return 0;
}

static int
virtio_crypto_sym_pad_auth_param(
		struct virtio_crypto_op_ctrl_req *ctrl,
		struct rte_crypto_auth_xform *auth_xform)
{
	uint32_t *algo;
	struct virtio_crypto_alg_chain_session_para *para =
		&(ctrl->u.sym_create_session.u.chain.para);

	switch (ctrl->u.sym_create_session.u.chain.para.hash_mode) {
	case VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN:
		algo = &(para->u.hash_param.algo);
		break;
	case VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH:
		algo = &(para->u.mac_param.algo);
		break;
	default:
		VIRTIO_CRYPTO_SESSION_LOG_ERR("Unsupported hash mode %u "
			"specified",
			ctrl->u.sym_create_session.u.chain.para.hash_mode);
		return -1;
	}

	switch (auth_xform->algo) {
	case RTE_CRYPTO_AUTH_SHA1_HMAC:
		*algo = VIRTIO_CRYPTO_MAC_HMAC_SHA1;
		break;
	default:
		VIRTIO_CRYPTO_SESSION_LOG_ERR(
			"Crypto: Undefined Hash algo %u specified",
			auth_xform->algo);
		return -1;
	}

	return 0;
}

static int
virtio_crypto_sym_pad_op_ctrl_req(
		struct virtio_crypto_op_ctrl_req *ctrl,
		struct rte_crypto_sym_xform *xform, bool is_chainned,
		uint8_t *cipher_key_data, uint8_t *auth_key_data,
		struct virtio_crypto_session *session)
{
	int ret;
	struct rte_crypto_auth_xform *auth_xform = NULL;
	struct rte_crypto_cipher_xform *cipher_xform = NULL;

	/* Get cipher xform from crypto xform chain */
	cipher_xform = virtio_crypto_get_cipher_xform(xform);
	if (cipher_xform) {
		if (cipher_xform->key.length > VIRTIO_CRYPTO_MAX_KEY_SIZE) {
			VIRTIO_CRYPTO_SESSION_LOG_ERR(
				"cipher key size cannot be longer than %u",
				VIRTIO_CRYPTO_MAX_KEY_SIZE);
			return -1;
		}
		if (cipher_xform->iv.length > VIRTIO_CRYPTO_MAX_IV_SIZE) {
			VIRTIO_CRYPTO_SESSION_LOG_ERR(
				"cipher IV size cannot be longer than %u",
				VIRTIO_CRYPTO_MAX_IV_SIZE);
			return -1;
		}
		if (is_chainned)
			ret = virtio_crypto_sym_pad_cipher_param(
				&ctrl->u.sym_create_session.u.chain.para
						.cipher_param, cipher_xform);
		else
			ret = virtio_crypto_sym_pad_cipher_param(
				&ctrl->u.sym_create_session.u.cipher.para,
				cipher_xform);

		if (ret < 0) {
			VIRTIO_CRYPTO_SESSION_LOG_ERR(
				"pad cipher parameter failed");
			return -1;
		}

		memcpy(cipher_key_data, cipher_xform->key.data,
				cipher_xform->key.length);

		session->iv.offset = cipher_xform->iv.offset;
		session->iv.length = cipher_xform->iv.length;
	}

	/* Get auth xform from crypto xform chain */
	auth_xform = virtio_crypto_get_auth_xform(xform);
	if (auth_xform) {
		/* FIXME: support VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */
		struct virtio_crypto_alg_chain_session_para *para =
			&(ctrl->u.sym_create_session.u.chain.para);
		if (auth_xform->key.length) {
			if (auth_xform->key.length >
					VIRTIO_CRYPTO_MAX_KEY_SIZE) {
				VIRTIO_CRYPTO_SESSION_LOG_ERR(
				"auth key size cannot be longer than %u",
					VIRTIO_CRYPTO_MAX_KEY_SIZE);
				return -1;
			}
			para->hash_mode = VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH;
			para->u.mac_param.auth_key_len =
				(uint32_t)auth_xform->key.length;
			para->u.mac_param.hash_result_len =
				auth_xform->digest_length;
			memcpy(auth_key_data, auth_xform->key.data,
					auth_xform->key.length);
		} else {
			para->hash_mode	= VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN;
			para->u.hash_param.hash_result_len =
				auth_xform->digest_length;
		}

		ret = virtio_crypto_sym_pad_auth_param(ctrl, auth_xform);
		if (ret < 0) {
			VIRTIO_CRYPTO_SESSION_LOG_ERR("pad auth parameter "
						"failed");
			return -1;
		}
	}

	return 0;
}

static int
virtio_crypto_check_sym_configure_session_paras(
		struct rte_cryptodev *dev,
		struct rte_crypto_sym_xform *xform,
		struct rte_cryptodev_sym_session *sym_sess,
		struct rte_mempool *mempool)
{
	if (unlikely(xform == NULL) || unlikely(sym_sess == NULL) ||
		unlikely(mempool == NULL)) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("NULL pointer");
		return -1;
	}

	if (virtio_crypto_check_sym_session_paras(dev) < 0)
		return -1;

	return 0;
}

static int
virtio_crypto_sym_configure_session(
		struct rte_cryptodev *dev,
		struct rte_crypto_sym_xform *xform,
		struct rte_cryptodev_sym_session *sess,
		struct rte_mempool *mempool)
{
	int ret;
	struct virtio_crypto_session crypto_sess;
	void *session_private = &crypto_sess;
	struct virtio_crypto_session *session;
	struct virtio_crypto_op_ctrl_req *ctrl_req;
	enum virtio_crypto_cmd_id cmd_id;
	uint8_t cipher_key_data[VIRTIO_CRYPTO_MAX_KEY_SIZE] = {0};
	uint8_t auth_key_data[VIRTIO_CRYPTO_MAX_KEY_SIZE] = {0};
	struct virtio_crypto_hw *hw;
	struct virtqueue *control_vq;

	PMD_INIT_FUNC_TRACE();

	ret = virtio_crypto_check_sym_configure_session_paras(dev, xform,
			sess, mempool);
	if (ret < 0) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR("Invalid parameters");
		return ret;
	}

	if (rte_mempool_get(mempool, &session_private)) {
		VIRTIO_CRYPTO_SESSION_LOG_ERR(
			"Couldn't get object from session mempool");
		return -ENOMEM;
	}

	session = (struct virtio_crypto_session *)session_private;
	memset(session, 0, sizeof(struct virtio_crypto_session));
	ctrl_req = &session->ctrl;
	ctrl_req->header.opcode = VIRTIO_CRYPTO_CIPHER_CREATE_SESSION;
	/* FIXME: support multiqueue */
	ctrl_req->header.queue_id = 0;

	hw = dev->data->dev_private;
	control_vq = hw->cvq;

	cmd_id = virtio_crypto_get_chain_order(xform);
	if (cmd_id == VIRTIO_CRYPTO_CMD_CIPHER_HASH)
		ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order
			= VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
	if (cmd_id == VIRTIO_CRYPTO_CMD_HASH_CIPHER)
		ctrl_req->u.sym_create_session.u.chain.para.alg_chain_order
			= VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;

	switch (cmd_id) {
	case VIRTIO_CRYPTO_CMD_CIPHER_HASH:
	case VIRTIO_CRYPTO_CMD_HASH_CIPHER:
		ctrl_req->u.sym_create_session.op_type
			= VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING;

		ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req,
			xform, true, cipher_key_data, auth_key_data, session);
		if (ret < 0) {
			VIRTIO_CRYPTO_SESSION_LOG_ERR(
				"padding sym op ctrl req failed");
			goto error_out;
		}
		ret = virtio_crypto_send_command(control_vq, ctrl_req,
			cipher_key_data, auth_key_data, session);
		if (ret < 0) {
			VIRTIO_CRYPTO_SESSION_LOG_ERR(
				"create session failed: %d", ret);
			goto error_out;
		}
		break;
	case VIRTIO_CRYPTO_CMD_CIPHER:
		ctrl_req->u.sym_create_session.op_type
			= VIRTIO_CRYPTO_SYM_OP_CIPHER;
		ret = virtio_crypto_sym_pad_op_ctrl_req(ctrl_req, xform,
			false, cipher_key_data, auth_key_data, session);
		if (ret < 0) {
			VIRTIO_CRYPTO_SESSION_LOG_ERR(
				"padding sym op ctrl req failed");
			goto error_out;
		}
		ret = virtio_crypto_send_command(control_vq, ctrl_req,
			cipher_key_data, NULL, session);
		if (ret < 0) {
			VIRTIO_CRYPTO_SESSION_LOG_ERR(
				"create session failed: %d", ret);
			goto error_out;
		}
		break;
	default:
		VIRTIO_CRYPTO_SESSION_LOG_ERR(
			"Unsupported operation chain order parameter");
		goto error_out;
	}

	set_sym_session_private_data(sess, dev->driver_id,
		session_private);

	return 0;

error_out:
	return -1;
}

static void
virtio_crypto_dev_info_get(struct rte_cryptodev *dev,
		struct rte_cryptodev_info *info)
{
	struct virtio_crypto_hw *hw = dev->data->dev_private;

	PMD_INIT_FUNC_TRACE();

	if (info != NULL) {
		info->driver_id = cryptodev_virtio_driver_id;
		info->feature_flags = dev->feature_flags;
		info->max_nb_queue_pairs = hw->max_dataqueues;
		/* No limit of number of sessions */
		info->sym.max_nb_sessions = 0;
		info->capabilities = hw->virtio_dev_capabilities;
	}
}

static int
crypto_virtio_pci_probe(
	struct rte_pci_driver *pci_drv __rte_unused,
	struct rte_pci_device *pci_dev)
{
	struct rte_cryptodev_pmd_init_params init_params = {
		.name = "",
		.socket_id = pci_dev->device.numa_node,
		.private_data_size = sizeof(struct virtio_crypto_hw)
	};
	char name[RTE_CRYPTODEV_NAME_MAX_LEN];

	VIRTIO_CRYPTO_DRV_LOG_DBG("Found Crypto device at %02x:%02x.%x",
			pci_dev->addr.bus,
			pci_dev->addr.devid,
			pci_dev->addr.function);

	rte_pci_device_name(&pci_dev->addr, name, sizeof(name));

	return crypto_virtio_create(name, pci_dev, &init_params);
}

static int
crypto_virtio_pci_remove(
	struct rte_pci_device *pci_dev __rte_unused)
{
	struct rte_cryptodev *cryptodev;
	char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];

	if (pci_dev == NULL)
		return -EINVAL;

	rte_pci_device_name(&pci_dev->addr, cryptodev_name,
			sizeof(cryptodev_name));

	cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name);
	if (cryptodev == NULL)
		return -ENODEV;

	return virtio_crypto_dev_uninit(cryptodev);
}

static struct rte_pci_driver rte_virtio_crypto_driver = {
	.id_table = pci_id_virtio_crypto_map,
	.drv_flags = 0,
	.probe = crypto_virtio_pci_probe,
	.remove = crypto_virtio_pci_remove
};

static struct cryptodev_driver virtio_crypto_drv;

RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_VIRTIO_PMD, rte_virtio_crypto_driver);
RTE_PMD_REGISTER_CRYPTO_DRIVER(virtio_crypto_drv,
	rte_virtio_crypto_driver.driver,
	cryptodev_virtio_driver_id);
RTE_LOG_REGISTER(virtio_crypto_logtype_init, pmd.crypto.virtio.init, NOTICE);
RTE_LOG_REGISTER(virtio_crypto_logtype_session, pmd.crypto.virtio.session,
		 NOTICE);
RTE_LOG_REGISTER(virtio_crypto_logtype_rx, pmd.crypto.virtio.rx, NOTICE);
RTE_LOG_REGISTER(virtio_crypto_logtype_tx, pmd.crypto.virtio.tx, NOTICE);
RTE_LOG_REGISTER(virtio_crypto_logtype_driver, pmd.crypto.virtio.driver,
		 NOTICE);