d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

vdpa/mlx5: support direct HW notifications

Browse Source 
Add support for the next 2 callbacks:
get_vfio_device_fd and get_notify_area.

This will allow direct HW doorbell ringing from guest and will save CPU
usage in host.

By this patch, the QEMU will map the physical address of the virtio
device in guest directly to the physical address of the HW device
doorbell.

The guest doorbell write is 2 bytes transaction while some Mellanox nics
support only 4 bytes transactions.

Remove ConnectX-5 and BF1 devices support which don't support 2B
doorbell writes for HW triggering.

Signed-off-by: Matan Azrad <matan@mellanox.com>
Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
This commit is contained in:
Matan Azrad 2020-03-24 14:24:35 +00:00 committed by Ferruh Yigit
parent 4cae722c1b
commit bd09eb7b61
2 changed files with 63 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
drivers/vdpa/mlx5/mlx5_vdpa.c
View File

@ -132,6 +132,29 @@ mlx5_vdpa_set_vring_state(int vid, int vring, int state)
return mlx5_vdpa_virtq_enable(virtq, state);
}
static int
mlx5_vdpa_direct_db_prepare(struct mlx5_vdpa_priv *priv)
{
int ret;
if (priv->direct_notifier) {
ret = rte_vhost_host_notifier_ctrl(priv->vid, false);
if (ret != 0) {
DRV_LOG(INFO, "Direct HW notifier FD cannot be "
"destroyed for device %d: %d.", priv->vid, ret);
return -1;
}
priv->direct_notifier = 0;
}
ret = rte_vhost_host_notifier_ctrl(priv->vid, true);
if (ret != 0)
DRV_LOG(INFO, "Direct HW notifier FD cannot be configured for"
" device %d: %d.", priv->vid, ret);
else
priv->direct_notifier = 1;
return 0;
}
static int
mlx5_vdpa_features_set(int vid)
{
@ -209,8 +232,9 @@ mlx5_vdpa_dev_config(int vid)
return -1;
}
priv->vid = vid;
if (mlx5_vdpa_mem_register(priv) || mlx5_vdpa_virtqs_prepare(priv) ||
mlx5_vdpa_steer_setup(priv) || mlx5_vdpa_cqe_event_setup(priv)) {
if (mlx5_vdpa_mem_register(priv) || mlx5_vdpa_direct_db_prepare(priv) ||
mlx5_vdpa_virtqs_prepare(priv) || mlx5_vdpa_steer_setup(priv) ||
mlx5_vdpa_cqe_event_setup(priv)) {
mlx5_vdpa_dev_close(vid);
return -1;
}
@ -218,6 +242,40 @@ mlx5_vdpa_dev_config(int vid)
return 0;
}
static int
mlx5_vdpa_get_device_fd(int vid)
{
int did = rte_vhost_get_vdpa_device_id(vid);
struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
if (priv == NULL) {
DRV_LOG(ERR, "Invalid device id: %d.", did);
return -EINVAL;
}
return priv->ctx->cmd_fd;
}
static int
mlx5_vdpa_get_notify_area(int vid, int qid, uint64_t *offset, uint64_t *size)
{
int did = rte_vhost_get_vdpa_device_id(vid);
struct mlx5_vdpa_priv *priv = mlx5_vdpa_find_priv_resource_by_did(did);
RTE_SET_USED(qid);
if (priv == NULL) {
DRV_LOG(ERR, "Invalid device id: %d.", did);
return -EINVAL;
}
if (!priv->var) {
DRV_LOG(ERR, "VAR was not created for device %d, is the device"
" configured?.", did);
return -EINVAL;
}
*offset = priv->var->mmap_off;
*size = priv->var->length;
return 0;
}
static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
.get_queue_num = mlx5_vdpa_get_queue_num,
.get_features = mlx5_vdpa_get_vdpa_features,
@ -228,8 +286,8 @@ static struct rte_vdpa_dev_ops mlx5_vdpa_ops = {
.set_features = mlx5_vdpa_features_set,
.migration_done = NULL,
.get_vfio_group_fd = NULL,
.get_vfio_device_fd = NULL,
.get_notify_area = NULL,
.get_vfio_device_fd = mlx5_vdpa_get_device_fd,
.get_notify_area = mlx5_vdpa_get_notify_area,
};
static struct ibv_device *
@ -518,14 +576,6 @@ mlx5_vdpa_pci_remove(struct rte_pci_device *pci_dev)
}
static const struct rte_pci_id mlx5_vdpa_pci_id_map[] = {
{
RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
PCI_DEVICE_ID_MELLANOX_CONNECTX5BF)
},
{
RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF)
},
{
RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
PCI_DEVICE_ID_MELLANOX_CONNECTX6)

1
drivers/vdpa/mlx5/mlx5_vdpa.h
View File

@ -100,6 +100,7 @@ struct mlx5_vdpa_steer {
struct mlx5_vdpa_priv {
TAILQ_ENTRY(mlx5_vdpa_priv) next;
uint8_t configured;
uint8_t direct_notifier; /* Whether direct notifier is on or off. */
int id; /* vDPA device id. */
int vid; /* vhost device id. */
struct ibv_context *ctx; /* Device context. */