crypto/virtio: support device init
This patch implements the initialization of the virtio crypto device. The virtio crypto device conforms to virtio-1.0, so this patch only supports modern mode operation. The cryptodev is created at the virtio crypto pci device probing stage. The function of virtio_crypto_pkt_tx_burst() is used to burst transfer packets and virtio_crypto_pkt_rx_burst() is used to burst receive packets. Signed-off-by: Jay Zhou <jianjay.zhou@huawei.com> Reviewed-by: Fan Zhang <roy.fan.zhang@intel.com>
This commit is contained in:
Jay Zhou
Pablo de Lara
parent
8769079ae0
commit
25500d4b80
@ -8,6 +8,10 @@ include $(RTE_SDK)/mk/rte.vars.mk
|
||||
#
|
||||
LIB = librte_pmd_virtio_crypto.a
|
||||
|
||||
#
|
||||
# include virtio_crypto.h
|
||||
#
|
||||
CFLAGS += -I$(RTE_SDK)/lib/librte_vhost
|
||||
CFLAGS += -O3
|
||||
CFLAGS += $(WERROR_FLAGS)
|
||||
|
||||
@ -18,6 +22,9 @@ LIBABIVER := 1
|
||||
#
|
||||
# all source are stored in SRCS-y
|
||||
#
|
||||
SRCS-$(CONFIG_RTE_LIBRTE_PMD_VIRTIO_CRYPTO) += virtqueue.c
|
||||
SRCS-$(CONFIG_RTE_LIBRTE_PMD_VIRTIO_CRYPTO) += virtio_pci.c
|
||||
SRCS-$(CONFIG_RTE_LIBRTE_PMD_VIRTIO_CRYPTO) += virtio_rxtx.c
|
||||
SRCS-$(CONFIG_RTE_LIBRTE_PMD_VIRTIO_CRYPTO) += virtio_cryptodev.c
|
||||
|
||||
# this lib depends upon:
|
||||
|
||||
@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
# Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
|
||||
|
||||
includes += include_directories('../../../lib/librte_vhost')
|
||||
deps += 'bus_pci'
|
||||
name = 'virtio_crypto'
|
||||
sources = files('virtio_cryptodev.c')
|
||||
sources = files('virtio_cryptodev.c', 'virtio_pci.c',
|
||||
'virtio_rxtx.c', 'virtqueue.c')
|
||||
|
||||
@ -3,28 +3,241 @@
|
||||
*/
|
||||
#include <rte_pci.h>
|
||||
#include <rte_bus_pci.h>
|
||||
#include <rte_cryptodev.h>
|
||||
#include <rte_cryptodev_pmd.h>
|
||||
#include <rte_eal.h>
|
||||
|
||||
#include "virtio_cryptodev.h"
|
||||
#include "virtqueue.h"
|
||||
|
||||
int virtio_crypto_logtype_init;
|
||||
int virtio_crypto_logtype_session;
|
||||
int virtio_crypto_logtype_rx;
|
||||
int virtio_crypto_logtype_tx;
|
||||
int virtio_crypto_logtype_driver;
|
||||
|
||||
/*
|
||||
* The set of PCI devices this driver supports
|
||||
*/
|
||||
static const struct rte_pci_id pci_id_virtio_crypto_map[] = {
|
||||
{ RTE_PCI_DEVICE(VIRTIO_CRYPTO_PCI_VENDORID,
|
||||
VIRTIO_CRYPTO_PCI_DEVICEID) },
|
||||
{ .vendor_id = 0, /* sentinel */ },
|
||||
};
|
||||
|
||||
uint8_t cryptodev_virtio_driver_id;
|
||||
|
||||
/*
|
||||
* dev_ops for virtio, bare necessities for basic operation
|
||||
*/
|
||||
static struct rte_cryptodev_ops virtio_crypto_dev_ops = {
|
||||
/* Device related operations */
|
||||
.dev_configure = NULL,
|
||||
.dev_start = NULL,
|
||||
.dev_stop = NULL,
|
||||
.dev_close = NULL,
|
||||
.dev_infos_get = NULL,
|
||||
|
||||
.stats_get = NULL,
|
||||
.stats_reset = NULL,
|
||||
|
||||
.queue_pair_setup = NULL,
|
||||
.queue_pair_release = NULL,
|
||||
.queue_pair_start = NULL,
|
||||
.queue_pair_stop = NULL,
|
||||
.queue_pair_count = NULL,
|
||||
|
||||
/* Crypto related operations */
|
||||
.session_get_size = NULL,
|
||||
.session_configure = NULL,
|
||||
.session_clear = NULL,
|
||||
.qp_attach_session = NULL,
|
||||
.qp_detach_session = NULL
|
||||
};
|
||||
|
||||
static int
|
||||
virtio_negotiate_features(struct virtio_crypto_hw *hw, uint64_t req_features)
|
||||
{
|
||||
uint64_t host_features;
|
||||
|
||||
PMD_INIT_FUNC_TRACE();
|
||||
|
||||
/* Prepare guest_features: feature that driver wants to support */
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("guest_features before negotiate = %" PRIx64,
|
||||
req_features);
|
||||
|
||||
/* Read device(host) feature bits */
|
||||
host_features = VTPCI_OPS(hw)->get_features(hw);
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("host_features before negotiate = %" PRIx64,
|
||||
host_features);
|
||||
|
||||
/*
|
||||
* Negotiate features: Subset of device feature bits are written back
|
||||
* guest feature bits.
|
||||
*/
|
||||
hw->guest_features = req_features;
|
||||
hw->guest_features = vtpci_cryptodev_negotiate_features(hw,
|
||||
host_features);
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("features after negotiate = %" PRIx64,
|
||||
hw->guest_features);
|
||||
|
||||
if (hw->modern) {
|
||||
if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) {
|
||||
VIRTIO_CRYPTO_INIT_LOG_ERR(
|
||||
"VIRTIO_F_VERSION_1 features is not enabled.");
|
||||
return -1;
|
||||
}
|
||||
vtpci_cryptodev_set_status(hw,
|
||||
VIRTIO_CONFIG_STATUS_FEATURES_OK);
|
||||
if (!(vtpci_cryptodev_get_status(hw) &
|
||||
VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
|
||||
VIRTIO_CRYPTO_INIT_LOG_ERR("failed to set FEATURES_OK "
|
||||
"status!");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
hw->req_guest_features = req_features;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* reset device and renegotiate features if needed */
|
||||
static int
|
||||
virtio_crypto_init_device(struct rte_cryptodev *cryptodev,
|
||||
uint64_t req_features)
|
||||
{
|
||||
struct virtio_crypto_hw *hw = cryptodev->data->dev_private;
|
||||
struct virtio_crypto_config local_config;
|
||||
struct virtio_crypto_config *config = &local_config;
|
||||
|
||||
PMD_INIT_FUNC_TRACE();
|
||||
|
||||
/* Reset the device although not necessary at startup */
|
||||
vtpci_cryptodev_reset(hw);
|
||||
|
||||
/* Tell the host we've noticed this device. */
|
||||
vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
|
||||
|
||||
/* Tell the host we've known how to drive the device. */
|
||||
vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
|
||||
if (virtio_negotiate_features(hw, req_features) < 0)
|
||||
return -1;
|
||||
|
||||
/* Get status of the device */
|
||||
vtpci_read_cryptodev_config(hw,
|
||||
offsetof(struct virtio_crypto_config, status),
|
||||
&config->status, sizeof(config->status));
|
||||
if (config->status != VIRTIO_CRYPTO_S_HW_READY) {
|
||||
VIRTIO_CRYPTO_DRV_LOG_ERR("accelerator hardware is "
|
||||
"not ready");
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Get number of data queues */
|
||||
vtpci_read_cryptodev_config(hw,
|
||||
offsetof(struct virtio_crypto_config, max_dataqueues),
|
||||
&config->max_dataqueues,
|
||||
sizeof(config->max_dataqueues));
|
||||
hw->max_dataqueues = config->max_dataqueues;
|
||||
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("hw->max_dataqueues=%d",
|
||||
hw->max_dataqueues);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* This function is based on probe() function
|
||||
* It returns 0 on success.
|
||||
*/
|
||||
static int
|
||||
crypto_virtio_create(const char *name, struct rte_pci_device *pci_dev,
|
||||
struct rte_cryptodev_pmd_init_params *init_params)
|
||||
{
|
||||
struct rte_cryptodev *cryptodev;
|
||||
struct virtio_crypto_hw *hw;
|
||||
|
||||
PMD_INIT_FUNC_TRACE();
|
||||
|
||||
cryptodev = rte_cryptodev_pmd_create(name, &pci_dev->device,
|
||||
init_params);
|
||||
if (cryptodev == NULL)
|
||||
return -ENODEV;
|
||||
|
||||
cryptodev->driver_id = cryptodev_virtio_driver_id;
|
||||
cryptodev->dev_ops = &virtio_crypto_dev_ops;
|
||||
|
||||
cryptodev->enqueue_burst = virtio_crypto_pkt_tx_burst;
|
||||
cryptodev->dequeue_burst = virtio_crypto_pkt_rx_burst;
|
||||
|
||||
cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
|
||||
RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING;
|
||||
|
||||
hw = cryptodev->data->dev_private;
|
||||
hw->dev_id = cryptodev->data->dev_id;
|
||||
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("dev %d vendorID=0x%x deviceID=0x%x",
|
||||
cryptodev->data->dev_id, pci_dev->id.vendor_id,
|
||||
pci_dev->id.device_id);
|
||||
|
||||
/* pci device init */
|
||||
if (vtpci_cryptodev_init(pci_dev, hw))
|
||||
return -1;
|
||||
|
||||
if (virtio_crypto_init_device(cryptodev,
|
||||
VIRTIO_CRYPTO_PMD_GUEST_FEATURES) < 0)
|
||||
return -1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
crypto_virtio_pci_probe(
|
||||
struct rte_pci_driver *pci_drv __rte_unused,
|
||||
struct rte_pci_device *pci_dev __rte_unused)
|
||||
struct rte_pci_device *pci_dev)
|
||||
{
|
||||
return 0;
|
||||
struct rte_cryptodev_pmd_init_params init_params = {
|
||||
.name = "",
|
||||
.socket_id = rte_socket_id(),
|
||||
.private_data_size = sizeof(struct virtio_crypto_hw),
|
||||
.max_nb_sessions = RTE_VIRTIO_CRYPTO_PMD_MAX_NB_SESSIONS
|
||||
};
|
||||
char name[RTE_CRYPTODEV_NAME_MAX_LEN];
|
||||
|
||||
VIRTIO_CRYPTO_DRV_LOG_DBG("Found Crypto device at %02x:%02x.%x",
|
||||
pci_dev->addr.bus,
|
||||
pci_dev->addr.devid,
|
||||
pci_dev->addr.function);
|
||||
|
||||
rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
|
||||
|
||||
return crypto_virtio_create(name, pci_dev, &init_params);
|
||||
}
|
||||
|
||||
static int
|
||||
crypto_virtio_pci_remove(
|
||||
struct rte_pci_device *pci_dev __rte_unused)
|
||||
{
|
||||
struct rte_cryptodev *cryptodev;
|
||||
char cryptodev_name[RTE_CRYPTODEV_NAME_MAX_LEN];
|
||||
|
||||
if (pci_dev == NULL)
|
||||
return -EINVAL;
|
||||
|
||||
rte_pci_device_name(&pci_dev->addr, cryptodev_name,
|
||||
sizeof(cryptodev_name));
|
||||
|
||||
cryptodev = rte_cryptodev_pmd_get_named_dev(cryptodev_name);
|
||||
if (cryptodev == NULL)
|
||||
return -ENODEV;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct rte_pci_driver rte_virtio_crypto_driver = {
|
||||
.id_table = pci_id_virtio_crypto_map,
|
||||
.drv_flags = 0,
|
||||
.probe = crypto_virtio_pci_probe,
|
||||
.remove = crypto_virtio_pci_remove
|
||||
};
|
||||
@ -35,3 +248,31 @@ RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_VIRTIO_PMD, rte_virtio_crypto_driver);
|
||||
RTE_PMD_REGISTER_CRYPTO_DRIVER(virtio_crypto_drv,
|
||||
rte_virtio_crypto_driver.driver,
|
||||
cryptodev_virtio_driver_id);
|
||||
|
||||
RTE_INIT(virtio_crypto_init_log);
|
||||
static void
|
||||
virtio_crypto_init_log(void)
|
||||
{
|
||||
virtio_crypto_logtype_init = rte_log_register("pmd.crypto.virtio.init");
|
||||
if (virtio_crypto_logtype_init >= 0)
|
||||
rte_log_set_level(virtio_crypto_logtype_init, RTE_LOG_NOTICE);
|
||||
|
||||
virtio_crypto_logtype_session =
|
||||
rte_log_register("pmd.crypto.virtio.session");
|
||||
if (virtio_crypto_logtype_session >= 0)
|
||||
rte_log_set_level(virtio_crypto_logtype_session,
|
||||
RTE_LOG_NOTICE);
|
||||
|
||||
virtio_crypto_logtype_rx = rte_log_register("pmd.crypto.virtio.rx");
|
||||
if (virtio_crypto_logtype_rx >= 0)
|
||||
rte_log_set_level(virtio_crypto_logtype_rx, RTE_LOG_NOTICE);
|
||||
|
||||
virtio_crypto_logtype_tx = rte_log_register("pmd.crypto.virtio.tx");
|
||||
if (virtio_crypto_logtype_tx >= 0)
|
||||
rte_log_set_level(virtio_crypto_logtype_tx, RTE_LOG_NOTICE);
|
||||
|
||||
virtio_crypto_logtype_driver =
|
||||
rte_log_register("pmd.crypto.virtio.driver");
|
||||
if (virtio_crypto_logtype_driver >= 0)
|
||||
rte_log_set_level(virtio_crypto_logtype_driver, RTE_LOG_NOTICE);
|
||||
}
|
||||
|
||||
@ -5,6 +5,19 @@
|
||||
#ifndef _VIRTIO_CRYPTODEV_H_
|
||||
#define _VIRTIO_CRYPTODEV_H_
|
||||
|
||||
#include <rte_cryptodev.h>
|
||||
|
||||
/* Features desired/implemented by this driver. */
|
||||
#define VIRTIO_CRYPTO_PMD_GUEST_FEATURES (1ULL << VIRTIO_F_VERSION_1)
|
||||
|
||||
#define CRYPTODEV_NAME_VIRTIO_PMD crypto_virtio
|
||||
|
||||
uint16_t virtio_crypto_pkt_tx_burst(void *tx_queue,
|
||||
struct rte_crypto_op **tx_pkts,
|
||||
uint16_t nb_pkts);
|
||||
|
||||
uint16_t virtio_crypto_pkt_rx_burst(void *tx_queue,
|
||||
struct rte_crypto_op **tx_pkts,
|
||||
uint16_t nb_pkts);
|
||||
|
||||
#endif /* _VIRTIO_CRYPTODEV_H_ */
|
||||
|
||||
91
drivers/crypto/virtio/virtio_logs.h
Normal file
91
drivers/crypto/virtio/virtio_logs.h
Normal file
@ -0,0 +1,91 @@
|
||||
/* SPDX-License-Identifier: BSD-3-Clause
|
||||
* Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
|
||||
*/
|
||||
|
||||
#ifndef _VIRTIO_LOGS_H_
|
||||
#define _VIRTIO_LOGS_H_
|
||||
|
||||
#include <rte_log.h>
|
||||
|
||||
#define PMD_INIT_LOG(level, fmt, args...) \
|
||||
rte_log(RTE_LOG_ ## level, RTE_LOGTYPE_PMD, \
|
||||
"PMD: %s(): " fmt "\n", __func__, ##args)
|
||||
|
||||
#define PMD_INIT_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
|
||||
|
||||
extern int virtio_crypto_logtype_init;
|
||||
|
||||
#define VIRTIO_CRYPTO_INIT_LOG_IMPL(level, fmt, args...) \
|
||||
rte_log(RTE_LOG_ ## level, virtio_crypto_logtype_init, \
|
||||
"INIT: %s(): " fmt "\n", __func__, ##args)
|
||||
|
||||
#define VIRTIO_CRYPTO_INIT_LOG_INFO(fmt, args...) \
|
||||
VIRTIO_CRYPTO_INIT_LOG_IMPL(INFO, fmt, ## args)
|
||||
|
||||
#define VIRTIO_CRYPTO_INIT_LOG_DBG(fmt, args...) \
|
||||
VIRTIO_CRYPTO_INIT_LOG_IMPL(DEBUG, fmt, ## args)
|
||||
|
||||
#define VIRTIO_CRYPTO_INIT_LOG_ERR(fmt, args...) \
|
||||
VIRTIO_CRYPTO_INIT_LOG_IMPL(ERR, fmt, ## args)
|
||||
|
||||
extern int virtio_crypto_logtype_session;
|
||||
|
||||
#define VIRTIO_CRYPTO_SESSION_LOG_IMPL(level, fmt, args...) \
|
||||
rte_log(RTE_LOG_ ## level, virtio_crypto_logtype_session, \
|
||||
"SESSION: %s(): " fmt "\n", __func__, ##args)
|
||||
|
||||
#define VIRTIO_CRYPTO_SESSION_LOG_INFO(fmt, args...) \
|
||||
VIRTIO_CRYPTO_SESSION_LOG_IMPL(INFO, fmt, ## args)
|
||||
|
||||
#define VIRTIO_CRYPTO_SESSION_LOG_DBG(fmt, args...) \
|
||||
VIRTIO_CRYPTO_SESSION_LOG_IMPL(DEBUG, fmt, ## args)
|
||||
|
||||
#define VIRTIO_CRYPTO_SESSION_LOG_ERR(fmt, args...) \
|
||||
VIRTIO_CRYPTO_SESSION_LOG_IMPL(ERR, fmt, ## args)
|
||||
|
||||
extern int virtio_crypto_logtype_rx;
|
||||
|
||||
#define VIRTIO_CRYPTO_RX_LOG_IMPL(level, fmt, args...) \
|
||||
rte_log(RTE_LOG_ ## level, virtio_crypto_logtype_rx, \
|
||||
"RX: %s(): " fmt "\n", __func__, ##args)
|
||||
|
||||
#define VIRTIO_CRYPTO_RX_LOG_INFO(fmt, args...) \
|
||||
VIRTIO_CRYPTO_RX_LOG_IMPL(INFO, fmt, ## args)
|
||||
|
||||
#define VIRTIO_CRYPTO_RX_LOG_DBG(fmt, args...) \
|
||||
VIRTIO_CRYPTO_RX_LOG_IMPL(DEBUG, fmt, ## args)
|
||||
|
||||
#define VIRTIO_CRYPTO_RX_LOG_ERR(fmt, args...) \
|
||||
VIRTIO_CRYPTO_RX_LOG_IMPL(ERR, fmt, ## args)
|
||||
|
||||
extern int virtio_crypto_logtype_tx;
|
||||
|
||||
#define VIRTIO_CRYPTO_TX_LOG_IMPL(level, fmt, args...) \
|
||||
rte_log(RTE_LOG_ ## level, virtio_crypto_logtype_tx, \
|
||||
"TX: %s(): " fmt "\n", __func__, ##args)
|
||||
|
||||
#define VIRTIO_CRYPTO_TX_LOG_INFO(fmt, args...) \
|
||||
VIRTIO_CRYPTO_TX_LOG_IMPL(INFO, fmt, ## args)
|
||||
|
||||
#define VIRTIO_CRYPTO_TX_LOG_DBG(fmt, args...) \
|
||||
VIRTIO_CRYPTO_TX_LOG_IMPL(DEBUG, fmt, ## args)
|
||||
|
||||
#define VIRTIO_CRYPTO_TX_LOG_ERR(fmt, args...) \
|
||||
VIRTIO_CRYPTO_TX_LOG_IMPL(ERR, fmt, ## args)
|
||||
|
||||
extern int virtio_crypto_logtype_driver;
|
||||
|
||||
#define VIRTIO_CRYPTO_DRV_LOG_IMPL(level, fmt, args...) \
|
||||
rte_log(RTE_LOG_ ## level, virtio_crypto_logtype_driver, \
|
||||
"DRIVER: %s(): " fmt "\n", __func__, ##args)
|
||||
|
||||
#define VIRTIO_CRYPTO_DRV_LOG_INFO(fmt, args...) \
|
||||
VIRTIO_CRYPTO_DRV_LOG_IMPL(INFO, fmt, ## args)
|
||||
|
||||
#define VIRTIO_CRYPTO_DRV_LOG_DBG(fmt, args...) \
|
||||
VIRTIO_CRYPTO_DRV_LOG_IMPL(DEBUG, fmt, ## args)
|
||||
|
||||
#define VIRTIO_CRYPTO_DRV_LOG_ERR(fmt, args...) \
|
||||
VIRTIO_CRYPTO_DRV_LOG_IMPL(ERR, fmt, ## args)
|
||||
|
||||
#endif /* _VIRTIO_LOGS_H_ */
|
||||
462
drivers/crypto/virtio/virtio_pci.c
Normal file
462
drivers/crypto/virtio/virtio_pci.c
Normal file
@ -0,0 +1,462 @@
|
||||
/* SPDX-License-Identifier: BSD-3-Clause
|
||||
* Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef RTE_EXEC_ENV_LINUXAPP
|
||||
#include <dirent.h>
|
||||
#include <fcntl.h>
|
||||
#endif
|
||||
|
||||
#include <rte_io.h>
|
||||
#include <rte_bus.h>
|
||||
|
||||
#include "virtio_pci.h"
|
||||
#include "virtqueue.h"
|
||||
|
||||
/*
|
||||
* Following macros are derived from linux/pci_regs.h, however,
|
||||
* we can't simply include that header here, as there is no such
|
||||
* file for non-Linux platform.
|
||||
*/
|
||||
#define PCI_CAPABILITY_LIST 0x34
|
||||
#define PCI_CAP_ID_VNDR 0x09
|
||||
#define PCI_CAP_ID_MSIX 0x11
|
||||
|
||||
/*
|
||||
* The remaining space is defined by each driver as the per-driver
|
||||
* configuration space.
|
||||
*/
|
||||
#define VIRTIO_PCI_CONFIG(hw) \
|
||||
(((hw)->use_msix == VIRTIO_MSIX_ENABLED) ? 24 : 20)
|
||||
|
||||
struct virtio_hw_internal virtio_hw_internal[RTE_MAX_VIRTIO_CRYPTO];
|
||||
|
||||
static inline int
|
||||
check_vq_phys_addr_ok(struct virtqueue *vq)
|
||||
{
|
||||
/* Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit,
|
||||
* and only accepts 32 bit page frame number.
|
||||
* Check if the allocated physical memory exceeds 16TB.
|
||||
*/
|
||||
if ((vq->vq_ring_mem + vq->vq_ring_size - 1) >>
|
||||
(VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
|
||||
VIRTIO_CRYPTO_INIT_LOG_ERR("vring address shouldn't be above 16TB!");
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static inline void
|
||||
io_write64_twopart(uint64_t val, uint32_t *lo, uint32_t *hi)
|
||||
{
|
||||
rte_write32(val & ((1ULL << 32) - 1), lo);
|
||||
rte_write32(val >> 32, hi);
|
||||
}
|
||||
|
||||
static void
|
||||
modern_read_dev_config(struct virtio_crypto_hw *hw, size_t offset,
|
||||
void *dst, int length)
|
||||
{
|
||||
int i;
|
||||
uint8_t *p;
|
||||
uint8_t old_gen, new_gen;
|
||||
|
||||
do {
|
||||
old_gen = rte_read8(&hw->common_cfg->config_generation);
|
||||
|
||||
p = dst;
|
||||
for (i = 0; i < length; i++)
|
||||
*p++ = rte_read8((uint8_t *)hw->dev_cfg + offset + i);
|
||||
|
||||
new_gen = rte_read8(&hw->common_cfg->config_generation);
|
||||
} while (old_gen != new_gen);
|
||||
}
|
||||
|
||||
static void
|
||||
modern_write_dev_config(struct virtio_crypto_hw *hw, size_t offset,
|
||||
const void *src, int length)
|
||||
{
|
||||
int i;
|
||||
const uint8_t *p = src;
|
||||
|
||||
for (i = 0; i < length; i++)
|
||||
rte_write8((*p++), (((uint8_t *)hw->dev_cfg) + offset + i));
|
||||
}
|
||||
|
||||
static uint64_t
|
||||
modern_get_features(struct virtio_crypto_hw *hw)
|
||||
{
|
||||
uint32_t features_lo, features_hi;
|
||||
|
||||
rte_write32(0, &hw->common_cfg->device_feature_select);
|
||||
features_lo = rte_read32(&hw->common_cfg->device_feature);
|
||||
|
||||
rte_write32(1, &hw->common_cfg->device_feature_select);
|
||||
features_hi = rte_read32(&hw->common_cfg->device_feature);
|
||||
|
||||
return ((uint64_t)features_hi << 32) | features_lo;
|
||||
}
|
||||
|
||||
static void
|
||||
modern_set_features(struct virtio_crypto_hw *hw, uint64_t features)
|
||||
{
|
||||
rte_write32(0, &hw->common_cfg->guest_feature_select);
|
||||
rte_write32(features & ((1ULL << 32) - 1),
|
||||
&hw->common_cfg->guest_feature);
|
||||
|
||||
rte_write32(1, &hw->common_cfg->guest_feature_select);
|
||||
rte_write32(features >> 32,
|
||||
&hw->common_cfg->guest_feature);
|
||||
}
|
||||
|
||||
static uint8_t
|
||||
modern_get_status(struct virtio_crypto_hw *hw)
|
||||
{
|
||||
return rte_read8(&hw->common_cfg->device_status);
|
||||
}
|
||||
|
||||
static void
|
||||
modern_set_status(struct virtio_crypto_hw *hw, uint8_t status)
|
||||
{
|
||||
rte_write8(status, &hw->common_cfg->device_status);
|
||||
}
|
||||
|
||||
static void
|
||||
modern_reset(struct virtio_crypto_hw *hw)
|
||||
{
|
||||
modern_set_status(hw, VIRTIO_CONFIG_STATUS_RESET);
|
||||
modern_get_status(hw);
|
||||
}
|
||||
|
||||
static uint8_t
|
||||
modern_get_isr(struct virtio_crypto_hw *hw)
|
||||
{
|
||||
return rte_read8(hw->isr);
|
||||
}
|
||||
|
||||
static uint16_t
|
||||
modern_set_config_irq(struct virtio_crypto_hw *hw, uint16_t vec)
|
||||
{
|
||||
rte_write16(vec, &hw->common_cfg->msix_config);
|
||||
return rte_read16(&hw->common_cfg->msix_config);
|
||||
}
|
||||
|
||||
static uint16_t
|
||||
modern_set_queue_irq(struct virtio_crypto_hw *hw, struct virtqueue *vq,
|
||||
uint16_t vec)
|
||||
{
|
||||
rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select);
|
||||
rte_write16(vec, &hw->common_cfg->queue_msix_vector);
|
||||
return rte_read16(&hw->common_cfg->queue_msix_vector);
|
||||
}
|
||||
|
||||
static uint16_t
|
||||
modern_get_queue_num(struct virtio_crypto_hw *hw, uint16_t queue_id)
|
||||
{
|
||||
rte_write16(queue_id, &hw->common_cfg->queue_select);
|
||||
return rte_read16(&hw->common_cfg->queue_size);
|
||||
}
|
||||
|
||||
static int
|
||||
modern_setup_queue(struct virtio_crypto_hw *hw, struct virtqueue *vq)
|
||||
{
|
||||
uint64_t desc_addr, avail_addr, used_addr;
|
||||
uint16_t notify_off;
|
||||
|
||||
if (!check_vq_phys_addr_ok(vq))
|
||||
return -1;
|
||||
|
||||
desc_addr = vq->vq_ring_mem;
|
||||
avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc);
|
||||
used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail,
|
||||
ring[vq->vq_nentries]),
|
||||
VIRTIO_PCI_VRING_ALIGN);
|
||||
|
||||
rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select);
|
||||
|
||||
io_write64_twopart(desc_addr, &hw->common_cfg->queue_desc_lo,
|
||||
&hw->common_cfg->queue_desc_hi);
|
||||
io_write64_twopart(avail_addr, &hw->common_cfg->queue_avail_lo,
|
||||
&hw->common_cfg->queue_avail_hi);
|
||||
io_write64_twopart(used_addr, &hw->common_cfg->queue_used_lo,
|
||||
&hw->common_cfg->queue_used_hi);
|
||||
|
||||
notify_off = rte_read16(&hw->common_cfg->queue_notify_off);
|
||||
vq->notify_addr = (void *)((uint8_t *)hw->notify_base +
|
||||
notify_off * hw->notify_off_multiplier);
|
||||
|
||||
rte_write16(1, &hw->common_cfg->queue_enable);
|
||||
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("queue %u addresses:", vq->vq_queue_index);
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("\t desc_addr: %" PRIx64, desc_addr);
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("\t aval_addr: %" PRIx64, avail_addr);
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("\t used_addr: %" PRIx64, used_addr);
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("\t notify addr: %p (notify offset: %u)",
|
||||
vq->notify_addr, notify_off);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void
|
||||
modern_del_queue(struct virtio_crypto_hw *hw, struct virtqueue *vq)
|
||||
{
|
||||
rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select);
|
||||
|
||||
io_write64_twopart(0, &hw->common_cfg->queue_desc_lo,
|
||||
&hw->common_cfg->queue_desc_hi);
|
||||
io_write64_twopart(0, &hw->common_cfg->queue_avail_lo,
|
||||
&hw->common_cfg->queue_avail_hi);
|
||||
io_write64_twopart(0, &hw->common_cfg->queue_used_lo,
|
||||
&hw->common_cfg->queue_used_hi);
|
||||
|
||||
rte_write16(0, &hw->common_cfg->queue_enable);
|
||||
}
|
||||
|
||||
static void
|
||||
modern_notify_queue(struct virtio_crypto_hw *hw __rte_unused,
|
||||
struct virtqueue *vq)
|
||||
{
|
||||
rte_write16(vq->vq_queue_index, vq->notify_addr);
|
||||
}
|
||||
|
||||
const struct virtio_pci_ops virtio_crypto_modern_ops = {
|
||||
.read_dev_cfg = modern_read_dev_config,
|
||||
.write_dev_cfg = modern_write_dev_config,
|
||||
.reset = modern_reset,
|
||||
.get_status = modern_get_status,
|
||||
.set_status = modern_set_status,
|
||||
.get_features = modern_get_features,
|
||||
.set_features = modern_set_features,
|
||||
.get_isr = modern_get_isr,
|
||||
.set_config_irq = modern_set_config_irq,
|
||||
.set_queue_irq = modern_set_queue_irq,
|
||||
.get_queue_num = modern_get_queue_num,
|
||||
.setup_queue = modern_setup_queue,
|
||||
.del_queue = modern_del_queue,
|
||||
.notify_queue = modern_notify_queue,
|
||||
};
|
||||
|
||||
void
|
||||
vtpci_read_cryptodev_config(struct virtio_crypto_hw *hw, size_t offset,
|
||||
void *dst, int length)
|
||||
{
|
||||
VTPCI_OPS(hw)->read_dev_cfg(hw, offset, dst, length);
|
||||
}
|
||||
|
||||
void
|
||||
vtpci_write_cryptodev_config(struct virtio_crypto_hw *hw, size_t offset,
|
||||
const void *src, int length)
|
||||
{
|
||||
VTPCI_OPS(hw)->write_dev_cfg(hw, offset, src, length);
|
||||
}
|
||||
|
||||
uint64_t
|
||||
vtpci_cryptodev_negotiate_features(struct virtio_crypto_hw *hw,
|
||||
uint64_t host_features)
|
||||
{
|
||||
uint64_t features;
|
||||
|
||||
/*
|
||||
* Limit negotiated features to what the driver, virtqueue, and
|
||||
* host all support.
|
||||
*/
|
||||
features = host_features & hw->guest_features;
|
||||
VTPCI_OPS(hw)->set_features(hw, features);
|
||||
|
||||
return features;
|
||||
}
|
||||
|
||||
void
|
||||
vtpci_cryptodev_reset(struct virtio_crypto_hw *hw)
|
||||
{
|
||||
VTPCI_OPS(hw)->set_status(hw, VIRTIO_CONFIG_STATUS_RESET);
|
||||
/* flush status write */
|
||||
VTPCI_OPS(hw)->get_status(hw);
|
||||
}
|
||||
|
||||
void
|
||||
vtpci_cryptodev_reinit_complete(struct virtio_crypto_hw *hw)
|
||||
{
|
||||
vtpci_cryptodev_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER_OK);
|
||||
}
|
||||
|
||||
void
|
||||
vtpci_cryptodev_set_status(struct virtio_crypto_hw *hw, uint8_t status)
|
||||
{
|
||||
if (status != VIRTIO_CONFIG_STATUS_RESET)
|
||||
status |= VTPCI_OPS(hw)->get_status(hw);
|
||||
|
||||
VTPCI_OPS(hw)->set_status(hw, status);
|
||||
}
|
||||
|
||||
uint8_t
|
||||
vtpci_cryptodev_get_status(struct virtio_crypto_hw *hw)
|
||||
{
|
||||
return VTPCI_OPS(hw)->get_status(hw);
|
||||
}
|
||||
|
||||
uint8_t
|
||||
vtpci_cryptodev_isr(struct virtio_crypto_hw *hw)
|
||||
{
|
||||
return VTPCI_OPS(hw)->get_isr(hw);
|
||||
}
|
||||
|
||||
static void *
|
||||
get_cfg_addr(struct rte_pci_device *dev, struct virtio_pci_cap *cap)
|
||||
{
|
||||
uint8_t bar = cap->bar;
|
||||
uint32_t length = cap->length;
|
||||
uint32_t offset = cap->offset;
|
||||
uint8_t *base;
|
||||
|
||||
if (bar >= PCI_MAX_RESOURCE) {
|
||||
VIRTIO_CRYPTO_INIT_LOG_ERR("invalid bar: %u", bar);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (offset + length < offset) {
|
||||
VIRTIO_CRYPTO_INIT_LOG_ERR("offset(%u) + length(%u) overflows",
|
||||
offset, length);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (offset + length > dev->mem_resource[bar].len) {
|
||||
VIRTIO_CRYPTO_INIT_LOG_ERR(
|
||||
"invalid cap: overflows bar space: %u > %" PRIu64,
|
||||
offset + length, dev->mem_resource[bar].len);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
base = dev->mem_resource[bar].addr;
|
||||
if (base == NULL) {
|
||||
VIRTIO_CRYPTO_INIT_LOG_ERR("bar %u base addr is NULL", bar);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return base + offset;
|
||||
}
|
||||
|
||||
#define PCI_MSIX_ENABLE 0x8000
|
||||
|
||||
static int
|
||||
virtio_read_caps(struct rte_pci_device *dev, struct virtio_crypto_hw *hw)
|
||||
{
|
||||
uint8_t pos;
|
||||
struct virtio_pci_cap cap;
|
||||
int ret;
|
||||
|
||||
if (rte_pci_map_device(dev)) {
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("failed to map pci device!");
|
||||
return -1;
|
||||
}
|
||||
|
||||
ret = rte_pci_read_config(dev, &pos, 1, PCI_CAPABILITY_LIST);
|
||||
if (ret < 0) {
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("failed to read pci capability list");
|
||||
return -1;
|
||||
}
|
||||
|
||||
while (pos) {
|
||||
ret = rte_pci_read_config(dev, &cap, sizeof(cap), pos);
|
||||
if (ret < 0) {
|
||||
VIRTIO_CRYPTO_INIT_LOG_ERR(
|
||||
"failed to read pci cap at pos: %x", pos);
|
||||
break;
|
||||
}
|
||||
|
||||
if (cap.cap_vndr == PCI_CAP_ID_MSIX) {
|
||||
/* Transitional devices would also have this capability,
|
||||
* that's why we also check if msix is enabled.
|
||||
* 1st byte is cap ID; 2nd byte is the position of next
|
||||
* cap; next two bytes are the flags.
|
||||
*/
|
||||
uint16_t flags = ((uint16_t *)&cap)[1];
|
||||
|
||||
if (flags & PCI_MSIX_ENABLE)
|
||||
hw->use_msix = VIRTIO_MSIX_ENABLED;
|
||||
else
|
||||
hw->use_msix = VIRTIO_MSIX_DISABLED;
|
||||
}
|
||||
|
||||
if (cap.cap_vndr != PCI_CAP_ID_VNDR) {
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG(
|
||||
"[%2x] skipping non VNDR cap id: %02x",
|
||||
pos, cap.cap_vndr);
|
||||
goto next;
|
||||
}
|
||||
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG(
|
||||
"[%2x] cfg type: %u, bar: %u, offset: %04x, len: %u",
|
||||
pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
|
||||
|
||||
switch (cap.cfg_type) {
|
||||
case VIRTIO_PCI_CAP_COMMON_CFG:
|
||||
hw->common_cfg = get_cfg_addr(dev, &cap);
|
||||
break;
|
||||
case VIRTIO_PCI_CAP_NOTIFY_CFG:
|
||||
rte_pci_read_config(dev, &hw->notify_off_multiplier,
|
||||
4, pos + sizeof(cap));
|
||||
hw->notify_base = get_cfg_addr(dev, &cap);
|
||||
break;
|
||||
case VIRTIO_PCI_CAP_DEVICE_CFG:
|
||||
hw->dev_cfg = get_cfg_addr(dev, &cap);
|
||||
break;
|
||||
case VIRTIO_PCI_CAP_ISR_CFG:
|
||||
hw->isr = get_cfg_addr(dev, &cap);
|
||||
break;
|
||||
}
|
||||
|
||||
next:
|
||||
pos = cap.cap_next;
|
||||
}
|
||||
|
||||
if (hw->common_cfg == NULL || hw->notify_base == NULL ||
|
||||
hw->dev_cfg == NULL || hw->isr == NULL) {
|
||||
VIRTIO_CRYPTO_INIT_LOG_INFO("no modern virtio pci device found.");
|
||||
return -1;
|
||||
}
|
||||
|
||||
VIRTIO_CRYPTO_INIT_LOG_INFO("found modern virtio pci device.");
|
||||
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("common cfg mapped at: %p", hw->common_cfg);
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("device cfg mapped at: %p", hw->dev_cfg);
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("isr cfg mapped at: %p", hw->isr);
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG("notify base: %p, notify off multiplier: %u",
|
||||
hw->notify_base, hw->notify_off_multiplier);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return -1:
|
||||
* if there is error mapping with VFIO/UIO.
|
||||
* if port map error when driver type is KDRV_NONE.
|
||||
* if whitelisted but driver type is KDRV_UNKNOWN.
|
||||
* Return 1 if kernel driver is managing the device.
|
||||
* Return 0 on success.
|
||||
*/
|
||||
int
|
||||
vtpci_cryptodev_init(struct rte_pci_device *dev, struct virtio_crypto_hw *hw)
|
||||
{
|
||||
/*
|
||||
* Try if we can succeed reading virtio pci caps, which exists
|
||||
* only on modern pci device. If failed, we fallback to legacy
|
||||
* virtio handling.
|
||||
*/
|
||||
if (virtio_read_caps(dev, hw) == 0) {
|
||||
VIRTIO_CRYPTO_INIT_LOG_INFO("modern virtio pci detected.");
|
||||
virtio_hw_internal[hw->dev_id].vtpci_ops =
|
||||
&virtio_crypto_modern_ops;
|
||||
hw->modern = 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* virtio crypto conforms to virtio 1.0 and doesn't support
|
||||
* legacy mode
|
||||
*/
|
||||
return -1;
|
||||
}
|
||||
253
drivers/crypto/virtio/virtio_pci.h
Normal file
253
drivers/crypto/virtio/virtio_pci.h
Normal file
@ -0,0 +1,253 @@
|
||||
/* SPDX-License-Identifier: BSD-3-Clause
|
||||
* Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
|
||||
*/
|
||||
|
||||
#ifndef _VIRTIO_PCI_H_
|
||||
#define _VIRTIO_PCI_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include <rte_pci.h>
|
||||
#include <rte_bus_pci.h>
|
||||
#include <rte_cryptodev.h>
|
||||
|
||||
#include "virtio_crypto.h"
|
||||
|
||||
struct virtqueue;
|
||||
|
||||
/* VirtIO PCI vendor/device ID. */
|
||||
#define VIRTIO_CRYPTO_PCI_VENDORID 0x1AF4
|
||||
#define VIRTIO_CRYPTO_PCI_DEVICEID 0x1054
|
||||
|
||||
/* VirtIO ABI version, this must match exactly. */
|
||||
#define VIRTIO_PCI_ABI_VERSION 0
|
||||
|
||||
/*
|
||||
* VirtIO Header, located in BAR 0.
|
||||
*/
|
||||
#define VIRTIO_PCI_HOST_FEATURES 0 /* host's supported features (32bit, RO)*/
|
||||
#define VIRTIO_PCI_GUEST_FEATURES 4 /* guest's supported features (32, RW) */
|
||||
#define VIRTIO_PCI_QUEUE_PFN 8 /* physical address of VQ (32, RW) */
|
||||
#define VIRTIO_PCI_QUEUE_NUM 12 /* number of ring entries (16, RO) */
|
||||
#define VIRTIO_PCI_QUEUE_SEL 14 /* current VQ selection (16, RW) */
|
||||
#define VIRTIO_PCI_QUEUE_NOTIFY 16 /* notify host regarding VQ (16, RW) */
|
||||
#define VIRTIO_PCI_STATUS 18 /* device status register (8, RW) */
|
||||
#define VIRTIO_PCI_ISR 19 /* interrupt status register, reading
|
||||
* also clears the register (8, RO)
|
||||
*/
|
||||
/* Only if MSIX is enabled: */
|
||||
|
||||
/* configuration change vector (16, RW) */
|
||||
#define VIRTIO_MSI_CONFIG_VECTOR 20
|
||||
/* vector for selected VQ notifications */
|
||||
#define VIRTIO_MSI_QUEUE_VECTOR 22
|
||||
|
||||
/* The bit of the ISR which indicates a device has an interrupt. */
|
||||
#define VIRTIO_PCI_ISR_INTR 0x1
|
||||
/* The bit of the ISR which indicates a device configuration change. */
|
||||
#define VIRTIO_PCI_ISR_CONFIG 0x2
|
||||
/* Vector value used to disable MSI for queue. */
|
||||
#define VIRTIO_MSI_NO_VECTOR 0xFFFF
|
||||
|
||||
/* Status byte for guest to report progress. */
|
||||
#define VIRTIO_CONFIG_STATUS_RESET 0x00
|
||||
#define VIRTIO_CONFIG_STATUS_ACK 0x01
|
||||
#define VIRTIO_CONFIG_STATUS_DRIVER 0x02
|
||||
#define VIRTIO_CONFIG_STATUS_DRIVER_OK 0x04
|
||||
#define VIRTIO_CONFIG_STATUS_FEATURES_OK 0x08
|
||||
#define VIRTIO_CONFIG_STATUS_FAILED 0x80
|
||||
|
||||
/*
|
||||
* Each virtqueue indirect descriptor list must be physically contiguous.
|
||||
* To allow us to malloc(9) each list individually, limit the number
|
||||
* supported to what will fit in one page. With 4KB pages, this is a limit
|
||||
* of 256 descriptors. If there is ever a need for more, we can switch to
|
||||
* contigmalloc(9) for the larger allocations, similar to what
|
||||
* bus_dmamem_alloc(9) does.
|
||||
*
|
||||
* Note the sizeof(struct vring_desc) is 16 bytes.
|
||||
*/
|
||||
#define VIRTIO_MAX_INDIRECT ((int) (PAGE_SIZE / 16))
|
||||
|
||||
/* Do we get callbacks when the ring is completely used, even if we've
|
||||
* suppressed them?
|
||||
*/
|
||||
#define VIRTIO_F_NOTIFY_ON_EMPTY 24
|
||||
|
||||
/* Can the device handle any descriptor layout? */
|
||||
#define VIRTIO_F_ANY_LAYOUT 27
|
||||
|
||||
/* We support indirect buffer descriptors */
|
||||
#define VIRTIO_RING_F_INDIRECT_DESC 28
|
||||
|
||||
#define VIRTIO_F_VERSION_1 32
|
||||
#define VIRTIO_F_IOMMU_PLATFORM 33
|
||||
|
||||
/* The Guest publishes the used index for which it expects an interrupt
|
||||
* at the end of the avail ring. Host should ignore the avail->flags field.
|
||||
*/
|
||||
/* The Host publishes the avail index for which it expects a kick
|
||||
* at the end of the used ring. Guest should ignore the used->flags field.
|
||||
*/
|
||||
#define VIRTIO_RING_F_EVENT_IDX 29
|
||||
|
||||
/* Common configuration */
|
||||
#define VIRTIO_PCI_CAP_COMMON_CFG 1
|
||||
/* Notifications */
|
||||
#define VIRTIO_PCI_CAP_NOTIFY_CFG 2
|
||||
/* ISR Status */
|
||||
#define VIRTIO_PCI_CAP_ISR_CFG 3
|
||||
/* Device specific configuration */
|
||||
#define VIRTIO_PCI_CAP_DEVICE_CFG 4
|
||||
/* PCI configuration access */
|
||||
#define VIRTIO_PCI_CAP_PCI_CFG 5
|
||||
|
||||
/* This is the PCI capability header: */
|
||||
struct virtio_pci_cap {
|
||||
uint8_t cap_vndr; /* Generic PCI field: PCI_CAP_ID_VNDR */
|
||||
uint8_t cap_next; /* Generic PCI field: next ptr. */
|
||||
uint8_t cap_len; /* Generic PCI field: capability length */
|
||||
uint8_t cfg_type; /* Identifies the structure. */
|
||||
uint8_t bar; /* Where to find it. */
|
||||
uint8_t padding[3]; /* Pad to full dword. */
|
||||
uint32_t offset; /* Offset within bar. */
|
||||
uint32_t length; /* Length of the structure, in bytes. */
|
||||
};
|
||||
|
||||
struct virtio_pci_notify_cap {
|
||||
struct virtio_pci_cap cap;
|
||||
uint32_t notify_off_multiplier; /* Multiplier for queue_notify_off. */
|
||||
};
|
||||
|
||||
/* Fields in VIRTIO_PCI_CAP_COMMON_CFG: */
|
||||
struct virtio_pci_common_cfg {
|
||||
/* About the whole device. */
|
||||
uint32_t device_feature_select; /* read-write */
|
||||
uint32_t device_feature; /* read-only */
|
||||
uint32_t guest_feature_select; /* read-write */
|
||||
uint32_t guest_feature; /* read-write */
|
||||
uint16_t msix_config; /* read-write */
|
||||
uint16_t num_queues; /* read-only */
|
||||
uint8_t device_status; /* read-write */
|
||||
uint8_t config_generation; /* read-only */
|
||||
|
||||
/* About a specific virtqueue. */
|
||||
uint16_t queue_select; /* read-write */
|
||||
uint16_t queue_size; /* read-write, power of 2. */
|
||||
uint16_t queue_msix_vector; /* read-write */
|
||||
uint16_t queue_enable; /* read-write */
|
||||
uint16_t queue_notify_off; /* read-only */
|
||||
uint32_t queue_desc_lo; /* read-write */
|
||||
uint32_t queue_desc_hi; /* read-write */
|
||||
uint32_t queue_avail_lo; /* read-write */
|
||||
uint32_t queue_avail_hi; /* read-write */
|
||||
uint32_t queue_used_lo; /* read-write */
|
||||
uint32_t queue_used_hi; /* read-write */
|
||||
};
|
||||
|
||||
struct virtio_crypto_hw;
|
||||
|
||||
struct virtio_pci_ops {
|
||||
void (*read_dev_cfg)(struct virtio_crypto_hw *hw, size_t offset,
|
||||
void *dst, int len);
|
||||
void (*write_dev_cfg)(struct virtio_crypto_hw *hw, size_t offset,
|
||||
const void *src, int len);
|
||||
void (*reset)(struct virtio_crypto_hw *hw);
|
||||
|
||||
uint8_t (*get_status)(struct virtio_crypto_hw *hw);
|
||||
void (*set_status)(struct virtio_crypto_hw *hw, uint8_t status);
|
||||
|
||||
uint64_t (*get_features)(struct virtio_crypto_hw *hw);
|
||||
void (*set_features)(struct virtio_crypto_hw *hw, uint64_t features);
|
||||
|
||||
uint8_t (*get_isr)(struct virtio_crypto_hw *hw);
|
||||
|
||||
uint16_t (*set_config_irq)(struct virtio_crypto_hw *hw, uint16_t vec);
|
||||
|
||||
uint16_t (*set_queue_irq)(struct virtio_crypto_hw *hw,
|
||||
struct virtqueue *vq, uint16_t vec);
|
||||
|
||||
uint16_t (*get_queue_num)(struct virtio_crypto_hw *hw,
|
||||
uint16_t queue_id);
|
||||
int (*setup_queue)(struct virtio_crypto_hw *hw, struct virtqueue *vq);
|
||||
void (*del_queue)(struct virtio_crypto_hw *hw, struct virtqueue *vq);
|
||||
void (*notify_queue)(struct virtio_crypto_hw *hw, struct virtqueue *vq);
|
||||
};
|
||||
|
||||
struct virtio_crypto_hw {
|
||||
/* control queue */
|
||||
struct virtqueue *cvq;
|
||||
uint16_t dev_id;
|
||||
uint16_t max_dataqueues;
|
||||
uint64_t req_guest_features;
|
||||
uint64_t guest_features;
|
||||
uint8_t use_msix;
|
||||
uint8_t modern;
|
||||
uint32_t notify_off_multiplier;
|
||||
uint8_t *isr;
|
||||
uint16_t *notify_base;
|
||||
struct virtio_pci_common_cfg *common_cfg;
|
||||
struct virtio_crypto_config *dev_cfg;
|
||||
const struct rte_cryptodev_capabilities *virtio_dev_capabilities;
|
||||
};
|
||||
|
||||
/*
|
||||
* While virtio_crypto_hw is stored in shared memory, this structure stores
|
||||
* some infos that may vary in the multiple process model locally.
|
||||
* For example, the vtpci_ops pointer.
|
||||
*/
|
||||
struct virtio_hw_internal {
|
||||
const struct virtio_pci_ops *vtpci_ops;
|
||||
struct rte_pci_ioport io;
|
||||
};
|
||||
|
||||
#define VTPCI_OPS(hw) (virtio_hw_internal[(hw)->dev_id].vtpci_ops)
|
||||
#define VTPCI_IO(hw) (&virtio_hw_internal[(hw)->dev_id].io)
|
||||
|
||||
extern struct virtio_hw_internal virtio_hw_internal[RTE_MAX_VIRTIO_CRYPTO];
|
||||
|
||||
/*
|
||||
* How many bits to shift physical queue address written to QUEUE_PFN.
|
||||
* 12 is historical, and due to x86 page size.
|
||||
*/
|
||||
#define VIRTIO_PCI_QUEUE_ADDR_SHIFT 12
|
||||
|
||||
/* The alignment to use between consumer and producer parts of vring. */
|
||||
#define VIRTIO_PCI_VRING_ALIGN 4096
|
||||
|
||||
enum virtio_msix_status {
|
||||
VIRTIO_MSIX_NONE = 0,
|
||||
VIRTIO_MSIX_DISABLED = 1,
|
||||
VIRTIO_MSIX_ENABLED = 2
|
||||
};
|
||||
|
||||
static inline int
|
||||
vtpci_with_feature(struct virtio_crypto_hw *hw, uint64_t bit)
|
||||
{
|
||||
return (hw->guest_features & (1ULL << bit)) != 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Function declaration from virtio_pci.c
|
||||
*/
|
||||
int vtpci_cryptodev_init(struct rte_pci_device *dev,
|
||||
struct virtio_crypto_hw *hw);
|
||||
void vtpci_cryptodev_reset(struct virtio_crypto_hw *hw);
|
||||
|
||||
void vtpci_cryptodev_reinit_complete(struct virtio_crypto_hw *hw);
|
||||
|
||||
uint8_t vtpci_cryptodev_get_status(struct virtio_crypto_hw *hw);
|
||||
void vtpci_cryptodev_set_status(struct virtio_crypto_hw *hw, uint8_t status);
|
||||
|
||||
uint64_t vtpci_cryptodev_negotiate_features(struct virtio_crypto_hw *hw,
|
||||
uint64_t host_features);
|
||||
|
||||
void vtpci_write_cryptodev_config(struct virtio_crypto_hw *hw, size_t offset,
|
||||
const void *src, int length);
|
||||
|
||||
void vtpci_read_cryptodev_config(struct virtio_crypto_hw *hw, size_t offset,
|
||||
void *dst, int length);
|
||||
|
||||
uint8_t vtpci_cryptodev_isr(struct virtio_crypto_hw *hw);
|
||||
|
||||
#endif /* _VIRTIO_PCI_H_ */
|
||||
137
drivers/crypto/virtio/virtio_ring.h
Normal file
137
drivers/crypto/virtio/virtio_ring.h
Normal file
@ -0,0 +1,137 @@
|
||||
/* SPDX-License-Identifier: BSD-3-Clause
|
||||
* Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
|
||||
*/
|
||||
|
||||
#ifndef _VIRTIO_RING_H_
|
||||
#define _VIRTIO_RING_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include <rte_common.h>
|
||||
|
||||
/* This marks a buffer as continuing via the next field. */
|
||||
#define VRING_DESC_F_NEXT 1
|
||||
/* This marks a buffer as write-only (otherwise read-only). */
|
||||
#define VRING_DESC_F_WRITE 2
|
||||
/* This means the buffer contains a list of buffer descriptors. */
|
||||
#define VRING_DESC_F_INDIRECT 4
|
||||
|
||||
/* The Host uses this in used->flags to advise the Guest: don't kick me
|
||||
* when you add a buffer. It's unreliable, so it's simply an
|
||||
* optimization. Guest will still kick if it's out of buffers.
|
||||
*/
|
||||
#define VRING_USED_F_NO_NOTIFY 1
|
||||
/* The Guest uses this in avail->flags to advise the Host: don't
|
||||
* interrupt me when you consume a buffer. It's unreliable, so it's
|
||||
* simply an optimization.
|
||||
*/
|
||||
#define VRING_AVAIL_F_NO_INTERRUPT 1
|
||||
|
||||
/* VirtIO ring descriptors: 16 bytes.
|
||||
* These can chain together via "next".
|
||||
*/
|
||||
struct vring_desc {
|
||||
uint64_t addr; /* Address (guest-physical). */
|
||||
uint32_t len; /* Length. */
|
||||
uint16_t flags; /* The flags as indicated above. */
|
||||
uint16_t next; /* We chain unused descriptors via this. */
|
||||
};
|
||||
|
||||
struct vring_avail {
|
||||
uint16_t flags;
|
||||
uint16_t idx;
|
||||
uint16_t ring[0];
|
||||
};
|
||||
|
||||
/* id is a 16bit index. uint32_t is used here for ids for padding reasons. */
|
||||
struct vring_used_elem {
|
||||
/* Index of start of used descriptor chain. */
|
||||
uint32_t id;
|
||||
/* Total length of the descriptor chain which was written to. */
|
||||
uint32_t len;
|
||||
};
|
||||
|
||||
struct vring_used {
|
||||
uint16_t flags;
|
||||
volatile uint16_t idx;
|
||||
struct vring_used_elem ring[0];
|
||||
};
|
||||
|
||||
struct vring {
|
||||
unsigned int num;
|
||||
struct vring_desc *desc;
|
||||
struct vring_avail *avail;
|
||||
struct vring_used *used;
|
||||
};
|
||||
|
||||
/* The standard layout for the ring is a continuous chunk of memory which
|
||||
* looks like this. We assume num is a power of 2.
|
||||
*
|
||||
* struct vring {
|
||||
* // The actual descriptors (16 bytes each)
|
||||
* struct vring_desc desc[num];
|
||||
*
|
||||
* // A ring of available descriptor heads with free-running index.
|
||||
* __u16 avail_flags;
|
||||
* __u16 avail_idx;
|
||||
* __u16 available[num];
|
||||
* __u16 used_event_idx;
|
||||
*
|
||||
* // Padding to the next align boundary.
|
||||
* char pad[];
|
||||
*
|
||||
* // A ring of used descriptor heads with free-running index.
|
||||
* __u16 used_flags;
|
||||
* __u16 used_idx;
|
||||
* struct vring_used_elem used[num];
|
||||
* __u16 avail_event_idx;
|
||||
* };
|
||||
*
|
||||
* NOTE: for VirtIO PCI, align is 4096.
|
||||
*/
|
||||
|
||||
/*
|
||||
* We publish the used event index at the end of the available ring, and vice
|
||||
* versa. They are at the end for backwards compatibility.
|
||||
*/
|
||||
#define vring_used_event(vr) ((vr)->avail->ring[(vr)->num])
|
||||
#define vring_avail_event(vr) (*(uint16_t *)&(vr)->used->ring[(vr)->num])
|
||||
|
||||
static inline size_t
|
||||
vring_size(unsigned int num, unsigned long align)
|
||||
{
|
||||
size_t size;
|
||||
|
||||
size = num * sizeof(struct vring_desc);
|
||||
size += sizeof(struct vring_avail) + (num * sizeof(uint16_t));
|
||||
size = RTE_ALIGN_CEIL(size, align);
|
||||
size += sizeof(struct vring_used) +
|
||||
(num * sizeof(struct vring_used_elem));
|
||||
return size;
|
||||
}
|
||||
|
||||
static inline void
|
||||
vring_init(struct vring *vr, unsigned int num, uint8_t *p,
|
||||
unsigned long align)
|
||||
{
|
||||
vr->num = num;
|
||||
vr->desc = (struct vring_desc *) p;
|
||||
vr->avail = (struct vring_avail *) (p +
|
||||
num * sizeof(struct vring_desc));
|
||||
vr->used = (void *)
|
||||
RTE_ALIGN_CEIL((uintptr_t)(&vr->avail->ring[num]), align);
|
||||
}
|
||||
|
||||
/*
|
||||
* The following is used with VIRTIO_RING_F_EVENT_IDX.
|
||||
* Assuming a given event_idx value from the other size, if we have
|
||||
* just incremented index from old to new_idx, should we trigger an
|
||||
* event?
|
||||
*/
|
||||
static inline int
|
||||
vring_need_event(uint16_t event_idx, uint16_t new_idx, uint16_t old)
|
||||
{
|
||||
return (uint16_t)(new_idx - event_idx - 1) < (uint16_t)(new_idx - old);
|
||||
}
|
||||
|
||||
#endif /* _VIRTIO_RING_H_ */
|
||||
26
drivers/crypto/virtio/virtio_rxtx.c
Normal file
26
drivers/crypto/virtio/virtio_rxtx.c
Normal file
@ -0,0 +1,26 @@
|
||||
/* SPDX-License-Identifier: BSD-3-Clause
|
||||
* Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
|
||||
*/
|
||||
#include "virtio_cryptodev.h"
|
||||
|
||||
uint16_t
|
||||
virtio_crypto_pkt_rx_burst(
|
||||
void *tx_queue __rte_unused,
|
||||
struct rte_crypto_op **rx_pkts __rte_unused,
|
||||
uint16_t nb_pkts __rte_unused)
|
||||
{
|
||||
uint16_t nb_rx = 0;
|
||||
|
||||
return nb_rx;
|
||||
}
|
||||
|
||||
uint16_t
|
||||
virtio_crypto_pkt_tx_burst(
|
||||
void *tx_queue __rte_unused,
|
||||
struct rte_crypto_op **tx_pkts __rte_unused,
|
||||
uint16_t nb_pkts __rte_unused)
|
||||
{
|
||||
uint16_t nb_tx = 0;
|
||||
|
||||
return nb_tx;
|
||||
}
|
||||
43
drivers/crypto/virtio/virtqueue.c
Normal file
43
drivers/crypto/virtio/virtqueue.c
Normal file
@ -0,0 +1,43 @@
|
||||
/* SPDX-License-Identifier: BSD-3-Clause
|
||||
* Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include <rte_mbuf.h>
|
||||
#include <rte_crypto.h>
|
||||
#include <rte_malloc.h>
|
||||
|
||||
#include "virtqueue.h"
|
||||
|
||||
void
|
||||
virtqueue_disable_intr(struct virtqueue *vq)
|
||||
{
|
||||
/*
|
||||
* Set VRING_AVAIL_F_NO_INTERRUPT to hint host
|
||||
* not to interrupt when it consumes packets
|
||||
* Note: this is only considered a hint to the host
|
||||
*/
|
||||
vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
|
||||
}
|
||||
|
||||
void
|
||||
virtqueue_detatch_unused(struct virtqueue *vq)
|
||||
{
|
||||
struct rte_crypto_op *cop = NULL;
|
||||
|
||||
int idx;
|
||||
|
||||
if (vq != NULL)
|
||||
for (idx = 0; idx < vq->vq_nentries; idx++) {
|
||||
cop = vq->vq_descx[idx].crypto_op;
|
||||
if (cop) {
|
||||
if (cop->sym->m_src)
|
||||
rte_pktmbuf_free(cop->sym->m_src);
|
||||
if (cop->sym->m_dst)
|
||||
rte_pktmbuf_free(cop->sym->m_dst);
|
||||
rte_crypto_op_free(cop);
|
||||
vq->vq_descx[idx].crypto_op = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
171
drivers/crypto/virtio/virtqueue.h
Normal file
171
drivers/crypto/virtio/virtqueue.h
Normal file
@ -0,0 +1,171 @@
|
||||
/* SPDX-License-Identifier: BSD-3-Clause
|
||||
* Copyright(c) 2018 HUAWEI TECHNOLOGIES CO., LTD.
|
||||
*/
|
||||
|
||||
#ifndef _VIRTQUEUE_H_
|
||||
#define _VIRTQUEUE_H_
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#include <rte_atomic.h>
|
||||
#include <rte_memory.h>
|
||||
#include <rte_memzone.h>
|
||||
#include <rte_mempool.h>
|
||||
|
||||
#include "virtio_pci.h"
|
||||
#include "virtio_ring.h"
|
||||
#include "virtio_logs.h"
|
||||
#include "virtio_crypto.h"
|
||||
|
||||
struct rte_mbuf;
|
||||
|
||||
/*
|
||||
* Per virtio_config.h in Linux.
|
||||
* For virtio_pci on SMP, we don't need to order with respect to MMIO
|
||||
* accesses through relaxed memory I/O windows, so smp_mb() et al are
|
||||
* sufficient.
|
||||
*
|
||||
*/
|
||||
#define virtio_mb() rte_smp_mb()
|
||||
#define virtio_rmb() rte_smp_rmb()
|
||||
#define virtio_wmb() rte_smp_wmb()
|
||||
|
||||
#define VIRTQUEUE_MAX_NAME_SZ 32
|
||||
|
||||
enum { VTCRYPTO_DATAQ = 0, VTCRYPTO_CTRLQ = 1 };
|
||||
|
||||
/**
|
||||
* The maximum virtqueue size is 2^15. Use that value as the end of
|
||||
* descriptor chain terminator since it will never be a valid index
|
||||
* in the descriptor table. This is used to verify we are correctly
|
||||
* handling vq_free_cnt.
|
||||
*/
|
||||
#define VQ_RING_DESC_CHAIN_END 32768
|
||||
|
||||
struct vq_desc_extra {
|
||||
void *crypto_op;
|
||||
void *cookie;
|
||||
uint16_t ndescs;
|
||||
};
|
||||
|
||||
struct virtqueue {
|
||||
/**< virtio_crypto_hw structure pointer. */
|
||||
struct virtio_crypto_hw *hw;
|
||||
/**< mem zone to populate RX ring. */
|
||||
const struct rte_memzone *mz;
|
||||
/**< memzone to populate hdr and request. */
|
||||
struct rte_mempool *mpool;
|
||||
uint8_t dev_id; /**< Device identifier. */
|
||||
uint16_t vq_queue_index; /**< PCI queue index */
|
||||
|
||||
void *vq_ring_virt_mem; /**< linear address of vring*/
|
||||
unsigned int vq_ring_size;
|
||||
phys_addr_t vq_ring_mem; /**< physical address of vring */
|
||||
|
||||
struct vring vq_ring; /**< vring keeping desc, used and avail */
|
||||
uint16_t vq_free_cnt; /**< num of desc available */
|
||||
uint16_t vq_nentries; /**< vring desc numbers */
|
||||
|
||||
/**
|
||||
* Head of the free chain in the descriptor table. If
|
||||
* there are no free descriptors, this will be set to
|
||||
* VQ_RING_DESC_CHAIN_END.
|
||||
*/
|
||||
uint16_t vq_desc_head_idx;
|
||||
uint16_t vq_desc_tail_idx;
|
||||
/**
|
||||
* Last consumed descriptor in the used table,
|
||||
* trails vq_ring.used->idx.
|
||||
*/
|
||||
uint16_t vq_used_cons_idx;
|
||||
uint16_t vq_avail_idx;
|
||||
|
||||
/* Statistics */
|
||||
uint64_t packets_sent_total;
|
||||
uint64_t packets_sent_failed;
|
||||
uint64_t packets_received_total;
|
||||
uint64_t packets_received_failed;
|
||||
|
||||
uint16_t *notify_addr;
|
||||
|
||||
struct vq_desc_extra vq_descx[0];
|
||||
};
|
||||
|
||||
/**
|
||||
* Tell the backend not to interrupt us.
|
||||
*/
|
||||
void virtqueue_disable_intr(struct virtqueue *vq);
|
||||
|
||||
/**
|
||||
* Get all mbufs to be freed.
|
||||
*/
|
||||
void virtqueue_detatch_unused(struct virtqueue *vq);
|
||||
|
||||
static inline int
|
||||
virtqueue_full(const struct virtqueue *vq)
|
||||
{
|
||||
return vq->vq_free_cnt == 0;
|
||||
}
|
||||
|
||||
#define VIRTQUEUE_NUSED(vq) \
|
||||
((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx))
|
||||
|
||||
static inline void
|
||||
vq_update_avail_idx(struct virtqueue *vq)
|
||||
{
|
||||
virtio_wmb();
|
||||
vq->vq_ring.avail->idx = vq->vq_avail_idx;
|
||||
}
|
||||
|
||||
static inline void
|
||||
vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx)
|
||||
{
|
||||
uint16_t avail_idx;
|
||||
/*
|
||||
* Place the head of the descriptor chain into the next slot and make
|
||||
* it usable to the host. The chain is made available now rather than
|
||||
* deferring to virtqueue_notify() in the hopes that if the host is
|
||||
* currently running on another CPU, we can keep it processing the new
|
||||
* descriptor.
|
||||
*/
|
||||
avail_idx = (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1));
|
||||
if (unlikely(vq->vq_ring.avail->ring[avail_idx] != desc_idx))
|
||||
vq->vq_ring.avail->ring[avail_idx] = desc_idx;
|
||||
vq->vq_avail_idx++;
|
||||
}
|
||||
|
||||
static inline int
|
||||
virtqueue_kick_prepare(struct virtqueue *vq)
|
||||
{
|
||||
return !(vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY);
|
||||
}
|
||||
|
||||
static inline void
|
||||
virtqueue_notify(struct virtqueue *vq)
|
||||
{
|
||||
/*
|
||||
* Ensure updated avail->idx is visible to host.
|
||||
* For virtio on IA, the notificaiton is through io port operation
|
||||
* which is a serialization instruction itself.
|
||||
*/
|
||||
VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq);
|
||||
}
|
||||
|
||||
/**
|
||||
* Dump virtqueue internal structures, for debug purpose only.
|
||||
*/
|
||||
#define VIRTQUEUE_DUMP(vq) do { \
|
||||
uint16_t used_idx, nused; \
|
||||
used_idx = (vq)->vq_ring.used->idx; \
|
||||
nused = (uint16_t)(used_idx - (vq)->vq_used_cons_idx); \
|
||||
VIRTIO_CRYPTO_INIT_LOG_DBG(\
|
||||
"VQ: - size=%d; free=%d; used=%d; desc_head_idx=%d;" \
|
||||
" avail.idx=%d; used_cons_idx=%d; used.idx=%d;" \
|
||||
" avail.flags=0x%x; used.flags=0x%x", \
|
||||
(vq)->vq_nentries, (vq)->vq_free_cnt, nused, \
|
||||
(vq)->vq_desc_head_idx, (vq)->vq_ring.avail->idx, \
|
||||
(vq)->vq_used_cons_idx, (vq)->vq_ring.used->idx, \
|
||||
(vq)->vq_ring.avail->flags, (vq)->vq_ring.used->flags); \
|
||||
} while (0)
|
||||
|
||||
#endif /* _VIRTQUEUE_H_ */
|
||||
Loading…
Reference in New Issue
Block a user