numam-dpdk/drivers/net/virtio/virtio_user/vhost_kernel.c

363 lines
9.1 KiB
C
Raw Normal View History

/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2016 Intel Corporation
*/
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <rte_memory.h>
#include <rte_eal_memconfig.h>
#include "vhost.h"
#include "virtio_user_dev.h"
#include "vhost_kernel_tap.h"
struct vhost_memory_kernel {
uint32_t nregions;
uint32_t padding;
struct vhost_memory_region regions[0];
};
/* vhost kernel ioctls */
#define VHOST_VIRTIO 0xAF
#define VHOST_GET_FEATURES _IOR(VHOST_VIRTIO, 0x00, __u64)
#define VHOST_SET_FEATURES _IOW(VHOST_VIRTIO, 0x00, __u64)
#define VHOST_SET_OWNER _IO(VHOST_VIRTIO, 0x01)
#define VHOST_RESET_OWNER _IO(VHOST_VIRTIO, 0x02)
#define VHOST_SET_MEM_TABLE _IOW(VHOST_VIRTIO, 0x03, struct vhost_memory_kernel)
#define VHOST_SET_LOG_BASE _IOW(VHOST_VIRTIO, 0x04, __u64)
#define VHOST_SET_LOG_FD _IOW(VHOST_VIRTIO, 0x07, int)
#define VHOST_SET_VRING_NUM _IOW(VHOST_VIRTIO, 0x10, struct vhost_vring_state)
#define VHOST_SET_VRING_ADDR _IOW(VHOST_VIRTIO, 0x11, struct vhost_vring_addr)
#define VHOST_SET_VRING_BASE _IOW(VHOST_VIRTIO, 0x12, struct vhost_vring_state)
#define VHOST_GET_VRING_BASE _IOWR(VHOST_VIRTIO, 0x12, struct vhost_vring_state)
#define VHOST_SET_VRING_KICK _IOW(VHOST_VIRTIO, 0x20, struct vhost_vring_file)
#define VHOST_SET_VRING_CALL _IOW(VHOST_VIRTIO, 0x21, struct vhost_vring_file)
#define VHOST_SET_VRING_ERR _IOW(VHOST_VIRTIO, 0x22, struct vhost_vring_file)
#define VHOST_NET_SET_BACKEND _IOW(VHOST_VIRTIO, 0x30, struct vhost_vring_file)
static uint64_t max_regions = 64;
static void
get_vhost_kernel_max_regions(void)
{
int fd;
char buf[20] = {'\0'};
fd = open("/sys/module/vhost/parameters/max_mem_regions", O_RDONLY);
if (fd < 0)
return;
if (read(fd, buf, sizeof(buf) - 1) > 0)
max_regions = strtoull(buf, NULL, 10);
close(fd);
}
static uint64_t vhost_req_user_to_kernel[] = {
[VHOST_USER_SET_OWNER] = VHOST_SET_OWNER,
[VHOST_USER_RESET_OWNER] = VHOST_RESET_OWNER,
[VHOST_USER_SET_FEATURES] = VHOST_SET_FEATURES,
[VHOST_USER_GET_FEATURES] = VHOST_GET_FEATURES,
[VHOST_USER_SET_VRING_CALL] = VHOST_SET_VRING_CALL,
[VHOST_USER_SET_VRING_NUM] = VHOST_SET_VRING_NUM,
[VHOST_USER_SET_VRING_BASE] = VHOST_SET_VRING_BASE,
[VHOST_USER_GET_VRING_BASE] = VHOST_GET_VRING_BASE,
[VHOST_USER_SET_VRING_ADDR] = VHOST_SET_VRING_ADDR,
[VHOST_USER_SET_VRING_KICK] = VHOST_SET_VRING_KICK,
[VHOST_USER_SET_MEM_TABLE] = VHOST_SET_MEM_TABLE,
};
struct walk_arg {
struct vhost_memory_kernel *vm;
uint32_t region_nr;
};
static int
mem: replace memseg with memseg lists Before, we were aggregating multiple pages into one memseg, so the number of memsegs was small. Now, each page gets its own memseg, so the list of memsegs is huge. To accommodate the new memseg list size and to keep the under-the-hood workings sane, the memseg list is now not just a single list, but multiple lists. To be precise, each hugepage size available on the system gets one or more memseg lists, per socket. In order to support dynamic memory allocation, we reserve all memory in advance (unless we're in 32-bit legacy mode, in which case we do not preallocate memory). As in, we do an anonymous mmap() of the entire maximum size of memory per hugepage size, per socket (which is limited to either RTE_MAX_MEMSEG_PER_TYPE pages or RTE_MAX_MEM_MB_PER_TYPE megabytes worth of memory, whichever is the smaller one), split over multiple lists (which are limited to either RTE_MAX_MEMSEG_PER_LIST memsegs or RTE_MAX_MEM_MB_PER_LIST megabytes per list, whichever is the smaller one). There is also a global limit of CONFIG_RTE_MAX_MEM_MB megabytes, which is mainly used for 32-bit targets to limit amounts of preallocated memory, but can be used to place an upper limit on total amount of VA memory that can be allocated by DPDK application. So, for each hugepage size, we get (by default) up to 128G worth of memory, per socket, split into chunks of up to 32G in size. The address space is claimed at the start, in eal_common_memory.c. The actual page allocation code is in eal_memalloc.c (Linux-only), and largely consists of copied EAL memory init code. Pages in the list are also indexed by address. That is, in order to figure out where the page belongs, one can simply look at base address for a memseg list. Similarly, figuring out IOVA address of a memzone is a matter of finding the right memseg list, getting offset and dividing by page size to get the appropriate memseg. This commit also removes rte_eal_dump_physmem_layout() call, according to deprecation notice [1], and removes that deprecation notice as well. On 32-bit targets due to limited VA space, DPDK will no longer spread memory to different sockets like before. Instead, it will (by default) allocate all of the memory on socket where master lcore is. To override this behavior, --socket-mem must be used. The rest of the changes are really ripple effects from the memseg change - heap changes, compile fixes, and rewrites to support fbarray-backed memseg lists. Due to earlier switch to _walk() functions, most of the changes are simple fixes, however some of the _walk() calls were switched to memseg list walk, where it made sense to do so. Additionally, we are also switching locks from flock() to fcntl(). Down the line, we will be introducing single-file segments option, and we cannot use flock() locks to lock parts of the file. Therefore, we will use fcntl() locks for legacy mem as well, in case someone is unfortunate enough to accidentally start legacy mem primary process alongside an already working non-legacy mem-based primary process. [1] http://dpdk.org/dev/patchwork/patch/34002/ Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com> Tested-by: Santosh Shukla <santosh.shukla@caviumnetworks.com> Tested-by: Hemant Agrawal <hemant.agrawal@nxp.com> Tested-by: Gowrishankar Muthukrishnan <gowrishankar.m@linux.vnet.ibm.com>
2018-04-11 12:30:24 +00:00
add_memory_region(const struct rte_memseg_list *msl __rte_unused,
const struct rte_memseg *ms, size_t len, void *arg)
{
struct walk_arg *wa = arg;
struct vhost_memory_region *mr;
void *start_addr;
if (wa->region_nr >= max_regions)
return -1;
mr = &wa->vm->regions[wa->region_nr++];
start_addr = ms->addr;
mr->guest_phys_addr = (uint64_t)(uintptr_t)start_addr;
mr->userspace_addr = (uint64_t)(uintptr_t)start_addr;
mr->memory_size = len;
mr->mmap_offset = 0;
return 0;
}
/* By default, vhost kernel module allows 64 regions, but DPDK allows
* 256 segments. As a relief, below function merges those virtually
* adjacent memsegs into one region.
*/
static struct vhost_memory_kernel *
prepare_vhost_memory_kernel(void)
{
struct vhost_memory_kernel *vm;
struct walk_arg wa;
vm = malloc(sizeof(struct vhost_memory_kernel) +
max_regions *
sizeof(struct vhost_memory_region));
if (!vm)
return NULL;
wa.region_nr = 0;
wa.vm = vm;
if (rte_memseg_contig_walk(add_memory_region, &wa) < 0) {
free(vm);
return NULL;
}
vm->nregions = wa.region_nr;
vm->padding = 0;
return vm;
}
/* with below features, vhost kernel does not need to do the checksum and TSO,
* these info will be passed to virtio_user through virtio net header.
*/
#define VHOST_KERNEL_GUEST_OFFLOADS_MASK \
((1ULL << VIRTIO_NET_F_GUEST_CSUM) | \
(1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
(1ULL << VIRTIO_NET_F_GUEST_TSO6) | \
(1ULL << VIRTIO_NET_F_GUEST_ECN) | \
(1ULL << VIRTIO_NET_F_GUEST_UFO))
/* with below features, when flows from virtio_user to vhost kernel
* (1) if flows goes up through the kernel networking stack, it does not need
* to verify checksum, which can save CPU cycles;
* (2) if flows goes through a Linux bridge and outside from an interface
* (kernel driver), checksum and TSO will be done by GSO in kernel or even
* offloaded into real physical device.
*/
#define VHOST_KERNEL_HOST_OFFLOADS_MASK \
((1ULL << VIRTIO_NET_F_HOST_TSO4) | \
(1ULL << VIRTIO_NET_F_HOST_TSO6) | \
(1ULL << VIRTIO_NET_F_CSUM))
static int
tap_supporte_mq(void)
{
int tapfd;
unsigned int tap_features;
tapfd = open(PATH_NET_TUN, O_RDWR);
if (tapfd < 0) {
PMD_DRV_LOG(ERR, "fail to open %s: %s",
PATH_NET_TUN, strerror(errno));
return -1;
}
if (ioctl(tapfd, TUNGETFEATURES, &tap_features) == -1) {
PMD_DRV_LOG(ERR, "TUNGETFEATURES failed: %s", strerror(errno));
close(tapfd);
return -1;
}
close(tapfd);
return tap_features & IFF_MULTI_QUEUE;
}
static int
vhost_kernel_ioctl(struct virtio_user_dev *dev,
enum vhost_user_request req,
void *arg)
{
int ret = -1;
unsigned int i;
uint64_t req_kernel;
struct vhost_memory_kernel *vm = NULL;
int vhostfd;
unsigned int queue_sel;
PMD_DRV_LOG(INFO, "%s", vhost_msg_strings[req]);
req_kernel = vhost_req_user_to_kernel[req];
if (req_kernel == VHOST_SET_MEM_TABLE) {
vm = prepare_vhost_memory_kernel();
if (!vm)
return -1;
arg = (void *)vm;
}
if (req_kernel == VHOST_SET_FEATURES) {
/* We don't need memory protection here */
*(uint64_t *)arg &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
/* VHOST kernel does not know about below flags */
*(uint64_t *)arg &= ~VHOST_KERNEL_GUEST_OFFLOADS_MASK;
*(uint64_t *)arg &= ~VHOST_KERNEL_HOST_OFFLOADS_MASK;
*(uint64_t *)arg &= ~(1ULL << VIRTIO_NET_F_MQ);
}
switch (req_kernel) {
case VHOST_SET_VRING_NUM:
case VHOST_SET_VRING_ADDR:
case VHOST_SET_VRING_BASE:
case VHOST_GET_VRING_BASE:
case VHOST_SET_VRING_KICK:
case VHOST_SET_VRING_CALL:
queue_sel = *(unsigned int *)arg;
vhostfd = dev->vhostfds[queue_sel / 2];
*(unsigned int *)arg = queue_sel % 2;
PMD_DRV_LOG(DEBUG, "vhostfd=%d, index=%u",
vhostfd, *(unsigned int *)arg);
break;
default:
vhostfd = -1;
}
if (vhostfd == -1) {
for (i = 0; i < dev->max_queue_pairs; ++i) {
if (dev->vhostfds[i] < 0)
continue;
ret = ioctl(dev->vhostfds[i], req_kernel, arg);
if (ret < 0)
break;
}
} else {
ret = ioctl(vhostfd, req_kernel, arg);
}
if (!ret && req_kernel == VHOST_GET_FEATURES) {
/* with tap as the backend, all these features are supported
* but not claimed by vhost-net, so we add them back when
* reporting to upper layer.
*/
*((uint64_t *)arg) |= VHOST_KERNEL_GUEST_OFFLOADS_MASK;
*((uint64_t *)arg) |= VHOST_KERNEL_HOST_OFFLOADS_MASK;
/* vhost_kernel will not declare this feature, but it does
* support multi-queue.
*/
if (tap_supporte_mq())
*(uint64_t *)arg |= (1ull << VIRTIO_NET_F_MQ);
}
if (vm)
free(vm);
if (ret < 0)
PMD_DRV_LOG(ERR, "%s failed: %s",
vhost_msg_strings[req], strerror(errno));
return ret;
}
/**
* Set up environment to talk with a vhost kernel backend.
*
* @return
* - (-1) if fail to set up;
* - (>=0) if successful.
*/
static int
vhost_kernel_setup(struct virtio_user_dev *dev)
{
int vhostfd;
uint32_t i;
get_vhost_kernel_max_regions();
for (i = 0; i < dev->max_queue_pairs; ++i) {
vhostfd = open(dev->path, O_RDWR);
if (vhostfd < 0) {
PMD_DRV_LOG(ERR, "fail to open %s, %s",
dev->path, strerror(errno));
return -1;
}
dev->vhostfds[i] = vhostfd;
}
return 0;
}
static int
vhost_kernel_set_backend(int vhostfd, int tapfd)
{
struct vhost_vring_file f;
f.fd = tapfd;
f.index = 0;
if (ioctl(vhostfd, VHOST_NET_SET_BACKEND, &f) < 0) {
PMD_DRV_LOG(ERR, "VHOST_NET_SET_BACKEND fails, %s",
strerror(errno));
return -1;
}
f.index = 1;
if (ioctl(vhostfd, VHOST_NET_SET_BACKEND, &f) < 0) {
PMD_DRV_LOG(ERR, "VHOST_NET_SET_BACKEND fails, %s",
strerror(errno));
return -1;
}
return 0;
}
static int
vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
uint16_t pair_idx,
int enable)
{
int hdr_size;
int vhostfd;
int tapfd;
int req_mq = (dev->max_queue_pairs > 1);
vhostfd = dev->vhostfds[pair_idx];
if (!enable) {
if (dev->tapfds[pair_idx] >= 0) {
close(dev->tapfds[pair_idx]);
dev->tapfds[pair_idx] = -1;
}
return vhost_kernel_set_backend(vhostfd, -1);
} else if (dev->tapfds[pair_idx] >= 0) {
return 0;
}
if ((dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF)) ||
(dev->features & (1ULL << VIRTIO_F_VERSION_1)))
hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
else
hdr_size = sizeof(struct virtio_net_hdr);
tapfd = vhost_kernel_open_tap(&dev->ifname, hdr_size, req_mq,
(char *)dev->mac_addr);
if (tapfd < 0) {
PMD_DRV_LOG(ERR, "fail to open tap for vhost kernel");
return -1;
}
if (vhost_kernel_set_backend(vhostfd, tapfd) < 0) {
PMD_DRV_LOG(ERR, "fail to set backend for vhost kernel");
close(tapfd);
return -1;
}
dev->tapfds[pair_idx] = tapfd;
return 0;
}
struct virtio_user_backend_ops ops_kernel = {
.setup = vhost_kernel_setup,
.send_request = vhost_kernel_ioctl,
.enable_qp = vhost_kernel_enable_queue_pair
};