d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

net/virtio-user: support server mode

Browse Source 
In a container environment if the vhost-user backend restarts, there's
no way for it to reconnect to virtio-user. To address this, support for
server mode is added. In this mode the socket file is created by virtio-
user, which the backend then connects to. This means that if the backend
restarts, it can reconnect to virtio-user and continue communications.

With current implementation, LSC is enabled at virtio-user side to
support to accept the coming connection.

Server mode virtio-user only supports to work with vhost-user.

Release note is updated in this patch.

Signed-off-by: Zhiyong Yang <zhiyong.yang@intel.com>
Reviewed-by: Jianfeng Tan <jianfeng.tan@intel.com>
This commit is contained in:
Zhiyong Yang 2018-04-06 17:25:54 +08:00 committed by Ferruh Yigit
parent f5188211c7
commit bd8f50a45d
5 changed files with 209 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
doc/guides/rel_notes/release_18_05.rst
View File

@ -69,6 +69,13 @@ New Features
See the :doc:`../nics/axgbe` nic driver guide for more details on this
new driver.
* **Added support for virtio-user server mode.**
In a container environment if the vhost-user backend restarts, there's no way
for it to reconnect to virtio-user. To address this, support for server mode
is added. In this mode the socket file is created by virtio-user, which the
backend connects to. This means that if the backend restarts, it can reconnect
to virtio-user and continue communications.
* **Added crypto workload support to vhost library.**
New APIs are introduced in vhost library to enable virtio crypto support

45
drivers/net/virtio/virtio_user/vhost_user.c
View File

@ -378,6 +378,30 @@ vhost_user_sock(struct virtio_user_dev *dev,
return 0;
}
#define MAX_VIRTIO_USER_BACKLOG 1
static int
virtio_user_start_server(struct virtio_user_dev *dev, struct sockaddr_un *un)
{
int ret;
int flag;
int fd = dev->listenfd;
ret = bind(fd, (struct sockaddr *)un, sizeof(*un));
if (ret < 0) {
PMD_DRV_LOG(ERR, "failed to bind to %s: %s; remove it and try again\n",
dev->path, strerror(errno));
return -1;
}
ret = listen(fd, MAX_VIRTIO_USER_BACKLOG);
if (ret < 0)
return -1;
flag = fcntl(fd, F_GETFL);
fcntl(fd, F_SETFL, flag | O_NONBLOCK);
return 0;
}
/**
* Set up environment to talk with a vhost user backend.
*
@ -405,13 +429,24 @@ vhost_user_setup(struct virtio_user_dev *dev)
memset(&un, 0, sizeof(un));
un.sun_family = AF_UNIX;
snprintf(un.sun_path, sizeof(un.sun_path), "%s", dev->path);
if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno));
close(fd);
return -1;
if (dev->is_server) {
dev->listenfd = fd;
if (virtio_user_start_server(dev, &un) < 0) {
PMD_DRV_LOG(ERR, "virtio-user startup fails in server mode");
close(fd);
return -1;
}
dev->vhostfd = -1;
} else {
if (connect(fd, (struct sockaddr *)&un, sizeof(un)) < 0) {
PMD_DRV_LOG(ERR, "connect error, %s", strerror(errno));
close(fd);
return -1;
}
dev->vhostfd = fd;
}
dev->vhostfd = fd;
return 0;
}

99
drivers/net/virtio/virtio_user/virtio_user_dev.c
View File

@ -93,12 +93,26 @@ virtio_user_queue_setup(struct virtio_user_dev *dev,
return 0;
}
int
is_vhost_user_by_type(const char *path)
{
struct stat sb;
if (stat(path, &sb) == -1)
return 0;
return S_ISSOCK(sb.st_mode);
}
int
virtio_user_start_device(struct virtio_user_dev *dev)
{
uint64_t features;
int ret;
if (is_vhost_user_by_type(dev->path) && dev->vhostfd < 0)
return -1;
/* Do not check return as already done in init, or reset in stop */
dev->ops->send_request(dev, VHOST_USER_SET_OWNER, NULL);
@ -174,17 +188,6 @@ parse_mac(struct virtio_user_dev *dev, const char *mac)
}
}
int
is_vhost_user_by_type(const char *path)
{
struct stat sb;
if (stat(path, &sb) == -1)
return 0;
return S_ISSOCK(sb.st_mode);
}
static int
virtio_user_dev_init_notify(struct virtio_user_dev *dev)
{
@ -254,6 +257,8 @@ virtio_user_fill_intr_handle(struct virtio_user_dev *dev)
eth_dev->intr_handle->fd = -1;
if (dev->vhostfd >= 0)
eth_dev->intr_handle->fd = dev->vhostfd;
else if (dev->is_server)
eth_dev->intr_handle->fd = dev->listenfd;
return 0;
}
@ -267,21 +272,32 @@ virtio_user_dev_setup(struct virtio_user_dev *dev)
dev->vhostfds = NULL;
dev->tapfds = NULL;
if (is_vhost_user_by_type(dev->path)) {
dev->ops = &ops_user;
} else {
dev->ops = &ops_kernel;
dev->vhostfds = malloc(dev->max_queue_pairs * sizeof(int));
dev->tapfds = malloc(dev->max_queue_pairs * sizeof(int));
if (!dev->vhostfds || !dev->tapfds) {
PMD_INIT_LOG(ERR, "Failed to malloc");
if (dev->is_server) {
if (access(dev->path, F_OK) == 0 &&
!is_vhost_user_by_type(dev->path)) {
PMD_DRV_LOG(ERR, "Server mode doesn't support vhost-kernel!");
return -1;
}
dev->ops = &ops_user;
} else {
if (is_vhost_user_by_type(dev->path)) {
dev->ops = &ops_user;
} else {
dev->ops = &ops_kernel;
for (q = 0; q < dev->max_queue_pairs; ++q) {
dev->vhostfds[q] = -1;
dev->tapfds[q] = -1;
dev->vhostfds = malloc(dev->max_queue_pairs *
sizeof(int));
dev->tapfds = malloc(dev->max_queue_pairs *
sizeof(int));
if (!dev->vhostfds || !dev->tapfds) {
PMD_INIT_LOG(ERR, "Failed to malloc");
return -1;
}
for (q = 0; q < dev->max_queue_pairs; ++q) {
dev->vhostfds[q] = -1;
dev->tapfds[q] = -1;
}
}
}
@ -337,16 +353,29 @@ virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues,
return -1;
}
if (dev->ops->send_request(dev, VHOST_USER_SET_OWNER, NULL) < 0) {
PMD_INIT_LOG(ERR, "set_owner fails: %s", strerror(errno));
return -1;
if (dev->vhostfd >= 0) {
if (dev->ops->send_request(dev, VHOST_USER_SET_OWNER,
NULL) < 0) {
PMD_INIT_LOG(ERR, "set_owner fails: %s",
strerror(errno));
return -1;
}
if (dev->ops->send_request(dev, VHOST_USER_GET_FEATURES,
&dev->device_features) < 0) {
PMD_INIT_LOG(ERR, "get_features failed: %s",
strerror(errno));
return -1;
}
} else {
/* We just pretend vhost-user can support all these features.
* Note that this could be problematic that if some feature is
* negotiated but not supported by the vhost-user which comes
* later.
*/
dev->device_features = VIRTIO_USER_SUPPORTED_FEATURES;
}
if (dev->ops->send_request(dev, VHOST_USER_GET_FEATURES,
&dev->device_features) < 0) {
PMD_INIT_LOG(ERR, "get_features failed: %s", strerror(errno));
return -1;
}
if (dev->mac_specified)
dev->device_features |= (1ull << VIRTIO_NET_F_MAC);
@ -388,6 +417,11 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
close(dev->vhostfd);
if (dev->is_server && dev->listenfd >= 0) {
close(dev->listenfd);
dev->listenfd = -1;
}
if (dev->vhostfds) {
for (i = 0; i < dev->max_queue_pairs; ++i)
close(dev->vhostfds[i]);
@ -396,6 +430,9 @@ virtio_user_dev_uninit(struct virtio_user_dev *dev)
}
free(dev->ifname);
if (dev->is_server)
unlink(dev->path);
}
static uint8_t

3
drivers/net/virtio/virtio_user/virtio_user_dev.h
View File

@ -6,6 +6,7 @@
#define _VIRTIO_USER_DEV_H
#include <limits.h>
#include <stdbool.h>
#include "../virtio_pci.h"
#include "../virtio_ring.h"
#include "vhost.h"
@ -13,6 +14,8 @@
struct virtio_user_dev {
/* for vhost_user backend */
int vhostfd;
int listenfd; /* listening fd */
bool is_server; /* server or client mode */
/* for vhost_kernel backend */
char *ifname;

101
drivers/net/virtio/virtio_user_ethdev.c
View File

@ -24,15 +24,73 @@
#define virtio_user_get_dev(hw) \
((struct virtio_user_dev *)(hw)->virtio_user_dev)
static int
virtio_user_server_reconnect(struct virtio_user_dev *dev)
{
int ret;
int flag;
int connectfd;
struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id];
connectfd = accept(dev->listenfd, NULL, NULL);
if (connectfd < 0)
return -1;
dev->vhostfd = connectfd;
flag = fcntl(connectfd, F_GETFD);
fcntl(connectfd, F_SETFL, flag | O_NONBLOCK);
ret = virtio_user_start_device(dev);
if (ret < 0)
return -1;
if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
if (rte_intr_disable(eth_dev->intr_handle) < 0) {
PMD_DRV_LOG(ERR, "interrupt disable failed");
return -1;
}
rte_intr_callback_unregister(eth_dev->intr_handle,
virtio_interrupt_handler,
eth_dev);
eth_dev->intr_handle->fd = connectfd;
rte_intr_callback_register(eth_dev->intr_handle,
virtio_interrupt_handler, eth_dev);
if (rte_intr_enable(eth_dev->intr_handle) < 0) {
PMD_DRV_LOG(ERR, "interrupt enable failed");
return -1;
}
}
PMD_INIT_LOG(NOTICE, "server mode virtio-user reconnection succeeds!");
return 0;
}
static void
virtio_user_delayed_handler(void *param)
{
struct virtio_hw *hw = (struct virtio_hw *)param;
struct rte_eth_dev *dev = &rte_eth_devices[hw->port_id];
struct rte_eth_dev *eth_dev = &rte_eth_devices[hw->port_id];
struct virtio_user_dev *dev = virtio_user_get_dev(hw);
rte_intr_callback_unregister(dev->intr_handle,
virtio_interrupt_handler,
dev);
if (rte_intr_disable(eth_dev->intr_handle) < 0) {
PMD_DRV_LOG(ERR, "interrupt disable failed");
return;
}
rte_intr_callback_unregister(eth_dev->intr_handle,
virtio_interrupt_handler, eth_dev);
if (dev->is_server) {
if (dev->vhostfd >= 0) {
close(dev->vhostfd);
dev->vhostfd = -1;
}
eth_dev->intr_handle->fd = dev->listenfd;
rte_intr_callback_register(eth_dev->intr_handle,
virtio_interrupt_handler, eth_dev);
if (rte_intr_enable(eth_dev->intr_handle) < 0) {
PMD_DRV_LOG(ERR, "interrupt enable failed");
return;
}
}
}
static void
@ -67,12 +125,10 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset,
dev->status &= (~VIRTIO_NET_S_LINK_UP);
PMD_DRV_LOG(ERR, "virtio-user port %u is down",
hw->port_id);
/* Only client mode is available now. Once the
* connection is broken, it can never be up
* again. Besides, this function could be called
* in the process of interrupt handling,
* callback cannot be unregistered here, set an
* alarm to do it.
/* This function could be called in the process
* of interrupt handling, callback cannot be
* unregistered here, set an alarm to do it.
*/
rte_eal_alarm_set(1,
virtio_user_delayed_handler,
@ -85,7 +141,12 @@ virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset,
PMD_DRV_LOG(ERR, "error clearing O_NONBLOCK flag");
return;
}
} else if (dev->is_server) {
dev->status &= (~VIRTIO_NET_S_LINK_UP);
if (virtio_user_server_reconnect(dev) >= 0)
dev->status |= VIRTIO_NET_S_LINK_UP;
}
*(uint16_t *)dst = dev->status;
}
@ -278,12 +339,15 @@ static const char *valid_args[] = {
VIRTIO_USER_ARG_QUEUE_SIZE,
#define VIRTIO_USER_ARG_INTERFACE_NAME "iface"
VIRTIO_USER_ARG_INTERFACE_NAME,
#define VIRTIO_USER_ARG_SERVER_MODE "server"
VIRTIO_USER_ARG_SERVER_MODE,
NULL
};
#define VIRTIO_USER_DEF_CQ_EN 0
#define VIRTIO_USER_DEF_Q_NUM 1
#define VIRTIO_USER_DEF_Q_SZ 256
#define VIRTIO_USER_DEF_SERVER_MODE 0
static int
get_string_arg(const char *key __rte_unused,
@ -378,6 +442,7 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
uint64_t queues = VIRTIO_USER_DEF_Q_NUM;
uint64_t cq = VIRTIO_USER_DEF_CQ_EN;
uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ;
uint64_t server_mode = VIRTIO_USER_DEF_SERVER_MODE;
char *path = NULL;
char *ifname = NULL;
char *mac_addr = NULL;
@ -445,6 +510,15 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
}
}
if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_SERVER_MODE) == 1) {
if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_SERVER_MODE,
&get_integer_arg, &server_mode) < 0) {
PMD_INIT_LOG(ERR, "error to parse %s",
VIRTIO_USER_ARG_SERVER_MODE);
goto end;
}
}
if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_CQ_NUM) == 1) {
if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM,
&get_integer_arg, &cq) < 0) {
@ -469,6 +543,8 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
}
if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
struct virtio_user_dev *vu_dev;
eth_dev = virtio_user_eth_dev_alloc(dev);
if (!eth_dev) {
PMD_INIT_LOG(ERR, "virtio_user fails to alloc device");
@ -476,6 +552,11 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev)
}
hw = eth_dev->data->dev_private;
vu_dev = virtio_user_get_dev(hw);
if (server_mode == 1)
vu_dev->is_server = true;
else
vu_dev->is_server = false;
if (virtio_user_dev_init(hw->virtio_user_dev, path, queues, cq,
queue_size, mac_addr, &ifname) < 0) {
PMD_INIT_LOG(ERR, "virtio_user_dev_init fails");