d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

vdpa/mlx5: avoid kick handling during shutdown

Browse Source 
When Qemu suspends a VM, HW notifier is un-mmapped while vCPU thread may
still be active and write notifier through kick socket.

PMD kick handler thread tries to install HW notifier through client
socket. In such case, it will timeout and slow down device close.

This patch skips HW notifier install if VQ or device in middle of
shutdown.

Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Reviewed-by: Maxime Coquelin <maxime.coquelin@redhat.com>
This commit is contained in:
Xueming Li 2022-05-08 17:25:50 +03:00 committed by Maxime Coquelin
parent 301ef4a185
commit b19cc62caf
3 changed files with 28 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
drivers/vdpa/mlx5/mlx5_vdpa.c
View File

@ -252,13 +252,15 @@ mlx5_vdpa_dev_close(int vid)
}
mlx5_vdpa_err_event_unset(priv);
mlx5_vdpa_cqe_event_unset(priv);
if (priv->configured)
if (priv->state == MLX5_VDPA_STATE_CONFIGURED) {
ret |= mlx5_vdpa_lm_log(priv);
priv->state = MLX5_VDPA_STATE_IN_PROGRESS;
}
mlx5_vdpa_steer_unset(priv);
mlx5_vdpa_virtqs_release(priv);
mlx5_vdpa_event_qp_global_release(priv);
mlx5_vdpa_mem_dereg(priv);
priv->configured = 0;
priv->state = MLX5_VDPA_STATE_PROBED;
priv->vid = 0;
/* The mutex may stay locked after event thread cancel - initiate it. */
pthread_mutex_init(&priv->vq_config_lock, NULL);
@ -277,7 +279,8 @@ mlx5_vdpa_dev_config(int vid)
DRV_LOG(ERR, "Invalid vDPA device: %s.", vdev->device->name);
return -EINVAL;
}
if (priv->configured && mlx5_vdpa_dev_close(vid)) {
if (priv->state == MLX5_VDPA_STATE_CONFIGURED &&
mlx5_vdpa_dev_close(vid)) {
DRV_LOG(ERR, "Failed to reconfigure vid %d.", vid);
return -1;
}
@ -291,7 +294,7 @@ mlx5_vdpa_dev_config(int vid)
mlx5_vdpa_dev_close(vid);
return -1;
}
priv->configured = 1;
priv->state = MLX5_VDPA_STATE_CONFIGURED;
DRV_LOG(INFO, "vDPA device %d was configured.", vid);
return 0;
}
@ -373,7 +376,7 @@ mlx5_vdpa_get_stats(struct rte_vdpa_device *vdev, int qid,
DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
return -ENODEV;
}
if (!priv->configured) {
if (priv->state == MLX5_VDPA_STATE_PROBED) {
DRV_LOG(ERR, "Device %s was not configured.",
vdev->device->name);
return -ENODATA;
@ -401,7 +404,7 @@ mlx5_vdpa_reset_stats(struct rte_vdpa_device *vdev, int qid)
DRV_LOG(ERR, "Invalid device: %s.", vdev->device->name);
return -ENODEV;
}
if (!priv->configured) {
if (priv->state == MLX5_VDPA_STATE_PROBED) {
DRV_LOG(ERR, "Device %s was not configured.",
vdev->device->name);
return -ENODATA;
@ -594,7 +597,7 @@ mlx5_vdpa_dev_remove(struct mlx5_common_device *cdev)
TAILQ_REMOVE(&priv_list, priv, next);
pthread_mutex_unlock(&priv_list_lock);
if (found) {
if (priv->configured)
if (priv->state == MLX5_VDPA_STATE_CONFIGURED)
mlx5_vdpa_dev_close(priv->vid);
if (priv->var) {
mlx5_glue->dv_free_var(priv->var);

8
drivers/vdpa/mlx5/mlx5_vdpa.h
View File

@ -113,9 +113,15 @@ enum {
MLX5_VDPA_EVENT_MODE_ONLY_INTERRUPT
};
enum mlx5_dev_state {
MLX5_VDPA_STATE_PROBED = 0,
MLX5_VDPA_STATE_CONFIGURED,
MLX5_VDPA_STATE_IN_PROGRESS /* Shutting down. */
};
struct mlx5_vdpa_priv {
TAILQ_ENTRY(mlx5_vdpa_priv) next;
uint8_t configured;
enum mlx5_dev_state state;
pthread_mutex_t vq_config_lock;
uint64_t no_traffic_counter;
pthread_t timer_tid;

12
drivers/vdpa/mlx5/mlx5_vdpa_virtq.c
View File

@ -25,6 +25,11 @@ mlx5_vdpa_virtq_kick_handler(void *cb_arg)
int nbytes;
int retry;
if (priv->state != MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) {
DRV_LOG(ERR, "device %d queue %d down, skip kick handling",
priv->vid, virtq->index);
return;
}
if (rte_intr_fd_get(virtq->intr_handle) < 0)
return;
for (retry = 0; retry < 3; ++retry) {
@ -43,6 +48,11 @@ mlx5_vdpa_virtq_kick_handler(void *cb_arg)
if (nbytes < 0)
return;
rte_write32(virtq->index, priv->virtq_db_addr);
if (priv->state != MLX5_VDPA_STATE_CONFIGURED && !virtq->enable) {
DRV_LOG(ERR, "device %d queue %d down, skip kick handling",
priv->vid, virtq->index);
return;
}
if (virtq->notifier_state == MLX5_VDPA_NOTIFIER_STATE_DISABLED) {
if (rte_vhost_host_notifier_ctrl(priv->vid, virtq->index, true))
virtq->notifier_state = MLX5_VDPA_NOTIFIER_STATE_ERR;
@ -541,7 +551,7 @@ mlx5_vdpa_virtq_enable(struct mlx5_vdpa_priv *priv, int index, int enable)
DRV_LOG(INFO, "Update virtq %d status %sable -> %sable.", index,
virtq->enable ? "en" : "dis", enable ? "en" : "dis");
if (!priv->configured) {
if (priv->state == MLX5_VDPA_STATE_PROBED) {
virtq->enable = !!enable;
return 0;
}