2017-12-19 15:49:01 +00:00
|
|
|
/* SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
* Copyright(c) 2016 Intel Corporation
|
2017-01-13 12:18:38 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <sys/types.h>
|
|
|
|
#include <sys/stat.h>
|
|
|
|
#include <fcntl.h>
|
|
|
|
#include <unistd.h>
|
|
|
|
|
|
|
|
#include <rte_memory.h>
|
|
|
|
#include <rte_eal_memconfig.h>
|
|
|
|
|
|
|
|
#include "vhost.h"
|
|
|
|
#include "virtio_user_dev.h"
|
|
|
|
#include "vhost_kernel_tap.h"
|
|
|
|
|
|
|
|
struct vhost_memory_kernel {
|
|
|
|
uint32_t nregions;
|
|
|
|
uint32_t padding;
|
|
|
|
struct vhost_memory_region regions[0];
|
|
|
|
};
|
|
|
|
|
|
|
|
/* vhost kernel ioctls */
|
|
|
|
#define VHOST_VIRTIO 0xAF
|
|
|
|
#define VHOST_GET_FEATURES _IOR(VHOST_VIRTIO, 0x00, __u64)
|
|
|
|
#define VHOST_SET_FEATURES _IOW(VHOST_VIRTIO, 0x00, __u64)
|
|
|
|
#define VHOST_SET_OWNER _IO(VHOST_VIRTIO, 0x01)
|
|
|
|
#define VHOST_RESET_OWNER _IO(VHOST_VIRTIO, 0x02)
|
|
|
|
#define VHOST_SET_MEM_TABLE _IOW(VHOST_VIRTIO, 0x03, struct vhost_memory_kernel)
|
|
|
|
#define VHOST_SET_LOG_BASE _IOW(VHOST_VIRTIO, 0x04, __u64)
|
|
|
|
#define VHOST_SET_LOG_FD _IOW(VHOST_VIRTIO, 0x07, int)
|
|
|
|
#define VHOST_SET_VRING_NUM _IOW(VHOST_VIRTIO, 0x10, struct vhost_vring_state)
|
|
|
|
#define VHOST_SET_VRING_ADDR _IOW(VHOST_VIRTIO, 0x11, struct vhost_vring_addr)
|
|
|
|
#define VHOST_SET_VRING_BASE _IOW(VHOST_VIRTIO, 0x12, struct vhost_vring_state)
|
|
|
|
#define VHOST_GET_VRING_BASE _IOWR(VHOST_VIRTIO, 0x12, struct vhost_vring_state)
|
|
|
|
#define VHOST_SET_VRING_KICK _IOW(VHOST_VIRTIO, 0x20, struct vhost_vring_file)
|
|
|
|
#define VHOST_SET_VRING_CALL _IOW(VHOST_VIRTIO, 0x21, struct vhost_vring_file)
|
|
|
|
#define VHOST_SET_VRING_ERR _IOW(VHOST_VIRTIO, 0x22, struct vhost_vring_file)
|
|
|
|
#define VHOST_NET_SET_BACKEND _IOW(VHOST_VIRTIO, 0x30, struct vhost_vring_file)
|
|
|
|
|
|
|
|
static uint64_t max_regions = 64;
|
|
|
|
|
|
|
|
static void
|
|
|
|
get_vhost_kernel_max_regions(void)
|
|
|
|
{
|
|
|
|
int fd;
|
|
|
|
char buf[20] = {'\0'};
|
|
|
|
|
|
|
|
fd = open("/sys/module/vhost/parameters/max_mem_regions", O_RDONLY);
|
|
|
|
if (fd < 0)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (read(fd, buf, sizeof(buf) - 1) > 0)
|
|
|
|
max_regions = strtoull(buf, NULL, 10);
|
|
|
|
|
|
|
|
close(fd);
|
|
|
|
}
|
|
|
|
|
|
|
|
static uint64_t vhost_req_user_to_kernel[] = {
|
|
|
|
[VHOST_USER_SET_OWNER] = VHOST_SET_OWNER,
|
|
|
|
[VHOST_USER_RESET_OWNER] = VHOST_RESET_OWNER,
|
|
|
|
[VHOST_USER_SET_FEATURES] = VHOST_SET_FEATURES,
|
|
|
|
[VHOST_USER_GET_FEATURES] = VHOST_GET_FEATURES,
|
|
|
|
[VHOST_USER_SET_VRING_CALL] = VHOST_SET_VRING_CALL,
|
|
|
|
[VHOST_USER_SET_VRING_NUM] = VHOST_SET_VRING_NUM,
|
|
|
|
[VHOST_USER_SET_VRING_BASE] = VHOST_SET_VRING_BASE,
|
|
|
|
[VHOST_USER_GET_VRING_BASE] = VHOST_GET_VRING_BASE,
|
|
|
|
[VHOST_USER_SET_VRING_ADDR] = VHOST_SET_VRING_ADDR,
|
|
|
|
[VHOST_USER_SET_VRING_KICK] = VHOST_SET_VRING_KICK,
|
|
|
|
[VHOST_USER_SET_MEM_TABLE] = VHOST_SET_MEM_TABLE,
|
|
|
|
};
|
|
|
|
|
2018-04-11 12:30:10 +00:00
|
|
|
struct walk_arg {
|
|
|
|
struct vhost_memory_kernel *vm;
|
|
|
|
uint32_t region_nr;
|
|
|
|
};
|
|
|
|
static int
|
mem: replace memseg with memseg lists
Before, we were aggregating multiple pages into one memseg, so the
number of memsegs was small. Now, each page gets its own memseg,
so the list of memsegs is huge. To accommodate the new memseg list
size and to keep the under-the-hood workings sane, the memseg list
is now not just a single list, but multiple lists. To be precise,
each hugepage size available on the system gets one or more memseg
lists, per socket.
In order to support dynamic memory allocation, we reserve all
memory in advance (unless we're in 32-bit legacy mode, in which
case we do not preallocate memory). As in, we do an anonymous
mmap() of the entire maximum size of memory per hugepage size, per
socket (which is limited to either RTE_MAX_MEMSEG_PER_TYPE pages or
RTE_MAX_MEM_MB_PER_TYPE megabytes worth of memory, whichever is the
smaller one), split over multiple lists (which are limited to
either RTE_MAX_MEMSEG_PER_LIST memsegs or RTE_MAX_MEM_MB_PER_LIST
megabytes per list, whichever is the smaller one). There is also
a global limit of CONFIG_RTE_MAX_MEM_MB megabytes, which is mainly
used for 32-bit targets to limit amounts of preallocated memory,
but can be used to place an upper limit on total amount of VA
memory that can be allocated by DPDK application.
So, for each hugepage size, we get (by default) up to 128G worth
of memory, per socket, split into chunks of up to 32G in size.
The address space is claimed at the start, in eal_common_memory.c.
The actual page allocation code is in eal_memalloc.c (Linux-only),
and largely consists of copied EAL memory init code.
Pages in the list are also indexed by address. That is, in order
to figure out where the page belongs, one can simply look at base
address for a memseg list. Similarly, figuring out IOVA address
of a memzone is a matter of finding the right memseg list, getting
offset and dividing by page size to get the appropriate memseg.
This commit also removes rte_eal_dump_physmem_layout() call,
according to deprecation notice [1], and removes that deprecation
notice as well.
On 32-bit targets due to limited VA space, DPDK will no longer
spread memory to different sockets like before. Instead, it will
(by default) allocate all of the memory on socket where master
lcore is. To override this behavior, --socket-mem must be used.
The rest of the changes are really ripple effects from the memseg
change - heap changes, compile fixes, and rewrites to support
fbarray-backed memseg lists. Due to earlier switch to _walk()
functions, most of the changes are simple fixes, however some
of the _walk() calls were switched to memseg list walk, where
it made sense to do so.
Additionally, we are also switching locks from flock() to fcntl().
Down the line, we will be introducing single-file segments option,
and we cannot use flock() locks to lock parts of the file. Therefore,
we will use fcntl() locks for legacy mem as well, in case someone is
unfortunate enough to accidentally start legacy mem primary process
alongside an already working non-legacy mem-based primary process.
[1] http://dpdk.org/dev/patchwork/patch/34002/
Signed-off-by: Anatoly Burakov <anatoly.burakov@intel.com>
Tested-by: Santosh Shukla <santosh.shukla@caviumnetworks.com>
Tested-by: Hemant Agrawal <hemant.agrawal@nxp.com>
Tested-by: Gowrishankar Muthukrishnan <gowrishankar.m@linux.vnet.ibm.com>
2018-04-11 12:30:24 +00:00
|
|
|
add_memory_region(const struct rte_memseg_list *msl __rte_unused,
|
|
|
|
const struct rte_memseg *ms, size_t len, void *arg)
|
2018-04-11 12:30:10 +00:00
|
|
|
{
|
|
|
|
struct walk_arg *wa = arg;
|
|
|
|
struct vhost_memory_region *mr;
|
|
|
|
void *start_addr;
|
|
|
|
|
|
|
|
if (wa->region_nr >= max_regions)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
mr = &wa->vm->regions[wa->region_nr++];
|
|
|
|
start_addr = ms->addr;
|
|
|
|
|
|
|
|
mr->guest_phys_addr = (uint64_t)(uintptr_t)start_addr;
|
|
|
|
mr->userspace_addr = (uint64_t)(uintptr_t)start_addr;
|
|
|
|
mr->memory_size = len;
|
|
|
|
mr->mmap_offset = 0;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-01-13 12:18:38 +00:00
|
|
|
/* By default, vhost kernel module allows 64 regions, but DPDK allows
|
|
|
|
* 256 segments. As a relief, below function merges those virtually
|
|
|
|
* adjacent memsegs into one region.
|
|
|
|
*/
|
|
|
|
static struct vhost_memory_kernel *
|
|
|
|
prepare_vhost_memory_kernel(void)
|
|
|
|
{
|
|
|
|
struct vhost_memory_kernel *vm;
|
2018-04-11 12:30:10 +00:00
|
|
|
struct walk_arg wa;
|
2017-01-13 12:18:38 +00:00
|
|
|
|
|
|
|
vm = malloc(sizeof(struct vhost_memory_kernel) +
|
2018-04-11 12:30:10 +00:00
|
|
|
max_regions *
|
|
|
|
sizeof(struct vhost_memory_region));
|
2017-01-26 03:05:42 +00:00
|
|
|
if (!vm)
|
|
|
|
return NULL;
|
2017-01-13 12:18:38 +00:00
|
|
|
|
2018-04-11 12:30:10 +00:00
|
|
|
wa.region_nr = 0;
|
|
|
|
wa.vm = vm;
|
|
|
|
|
|
|
|
if (rte_memseg_contig_walk(add_memory_region, &wa) < 0) {
|
|
|
|
free(vm);
|
|
|
|
return NULL;
|
2017-01-13 12:18:38 +00:00
|
|
|
}
|
|
|
|
|
2018-04-11 12:30:10 +00:00
|
|
|
vm->nregions = wa.region_nr;
|
2017-01-13 12:18:38 +00:00
|
|
|
vm->padding = 0;
|
|
|
|
return vm;
|
|
|
|
}
|
|
|
|
|
2017-01-13 12:18:39 +00:00
|
|
|
/* with below features, vhost kernel does not need to do the checksum and TSO,
|
|
|
|
* these info will be passed to virtio_user through virtio net header.
|
|
|
|
*/
|
|
|
|
#define VHOST_KERNEL_GUEST_OFFLOADS_MASK \
|
|
|
|
((1ULL << VIRTIO_NET_F_GUEST_CSUM) | \
|
|
|
|
(1ULL << VIRTIO_NET_F_GUEST_TSO4) | \
|
|
|
|
(1ULL << VIRTIO_NET_F_GUEST_TSO6) | \
|
|
|
|
(1ULL << VIRTIO_NET_F_GUEST_ECN) | \
|
|
|
|
(1ULL << VIRTIO_NET_F_GUEST_UFO))
|
|
|
|
|
|
|
|
/* with below features, when flows from virtio_user to vhost kernel
|
|
|
|
* (1) if flows goes up through the kernel networking stack, it does not need
|
|
|
|
* to verify checksum, which can save CPU cycles;
|
|
|
|
* (2) if flows goes through a Linux bridge and outside from an interface
|
|
|
|
* (kernel driver), checksum and TSO will be done by GSO in kernel or even
|
|
|
|
* offloaded into real physical device.
|
|
|
|
*/
|
|
|
|
#define VHOST_KERNEL_HOST_OFFLOADS_MASK \
|
|
|
|
((1ULL << VIRTIO_NET_F_HOST_TSO4) | \
|
|
|
|
(1ULL << VIRTIO_NET_F_HOST_TSO6) | \
|
|
|
|
(1ULL << VIRTIO_NET_F_CSUM))
|
|
|
|
|
2017-01-13 12:18:40 +00:00
|
|
|
static int
|
|
|
|
tap_supporte_mq(void)
|
|
|
|
{
|
|
|
|
int tapfd;
|
|
|
|
unsigned int tap_features;
|
|
|
|
|
|
|
|
tapfd = open(PATH_NET_TUN, O_RDWR);
|
|
|
|
if (tapfd < 0) {
|
|
|
|
PMD_DRV_LOG(ERR, "fail to open %s: %s",
|
|
|
|
PATH_NET_TUN, strerror(errno));
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (ioctl(tapfd, TUNGETFEATURES, &tap_features) == -1) {
|
|
|
|
PMD_DRV_LOG(ERR, "TUNGETFEATURES failed: %s", strerror(errno));
|
|
|
|
close(tapfd);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
close(tapfd);
|
|
|
|
return tap_features & IFF_MULTI_QUEUE;
|
|
|
|
}
|
|
|
|
|
2017-01-13 12:18:38 +00:00
|
|
|
static int
|
|
|
|
vhost_kernel_ioctl(struct virtio_user_dev *dev,
|
|
|
|
enum vhost_user_request req,
|
|
|
|
void *arg)
|
|
|
|
{
|
|
|
|
int ret = -1;
|
|
|
|
unsigned int i;
|
|
|
|
uint64_t req_kernel;
|
|
|
|
struct vhost_memory_kernel *vm = NULL;
|
2017-01-13 12:18:40 +00:00
|
|
|
int vhostfd;
|
|
|
|
unsigned int queue_sel;
|
2017-01-13 12:18:38 +00:00
|
|
|
|
|
|
|
PMD_DRV_LOG(INFO, "%s", vhost_msg_strings[req]);
|
|
|
|
|
|
|
|
req_kernel = vhost_req_user_to_kernel[req];
|
|
|
|
|
|
|
|
if (req_kernel == VHOST_SET_MEM_TABLE) {
|
|
|
|
vm = prepare_vhost_memory_kernel();
|
|
|
|
if (!vm)
|
|
|
|
return -1;
|
|
|
|
arg = (void *)vm;
|
|
|
|
}
|
|
|
|
|
2017-01-13 12:18:39 +00:00
|
|
|
if (req_kernel == VHOST_SET_FEATURES) {
|
|
|
|
/* We don't need memory protection here */
|
2017-01-13 12:18:38 +00:00
|
|
|
*(uint64_t *)arg &= ~(1ULL << VIRTIO_F_IOMMU_PLATFORM);
|
|
|
|
|
2017-01-13 12:18:39 +00:00
|
|
|
/* VHOST kernel does not know about below flags */
|
|
|
|
*(uint64_t *)arg &= ~VHOST_KERNEL_GUEST_OFFLOADS_MASK;
|
|
|
|
*(uint64_t *)arg &= ~VHOST_KERNEL_HOST_OFFLOADS_MASK;
|
2017-01-13 12:18:40 +00:00
|
|
|
|
|
|
|
*(uint64_t *)arg &= ~(1ULL << VIRTIO_NET_F_MQ);
|
2017-01-13 12:18:39 +00:00
|
|
|
}
|
|
|
|
|
2017-01-13 12:18:40 +00:00
|
|
|
switch (req_kernel) {
|
|
|
|
case VHOST_SET_VRING_NUM:
|
|
|
|
case VHOST_SET_VRING_ADDR:
|
|
|
|
case VHOST_SET_VRING_BASE:
|
|
|
|
case VHOST_GET_VRING_BASE:
|
|
|
|
case VHOST_SET_VRING_KICK:
|
|
|
|
case VHOST_SET_VRING_CALL:
|
|
|
|
queue_sel = *(unsigned int *)arg;
|
|
|
|
vhostfd = dev->vhostfds[queue_sel / 2];
|
|
|
|
*(unsigned int *)arg = queue_sel % 2;
|
|
|
|
PMD_DRV_LOG(DEBUG, "vhostfd=%d, index=%u",
|
|
|
|
vhostfd, *(unsigned int *)arg);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
vhostfd = -1;
|
|
|
|
}
|
|
|
|
if (vhostfd == -1) {
|
|
|
|
for (i = 0; i < dev->max_queue_pairs; ++i) {
|
|
|
|
if (dev->vhostfds[i] < 0)
|
|
|
|
continue;
|
2017-01-13 12:18:38 +00:00
|
|
|
|
2017-01-13 12:18:40 +00:00
|
|
|
ret = ioctl(dev->vhostfds[i], req_kernel, arg);
|
|
|
|
if (ret < 0)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
ret = ioctl(vhostfd, req_kernel, arg);
|
2017-01-13 12:18:38 +00:00
|
|
|
}
|
|
|
|
|
2017-01-13 12:18:39 +00:00
|
|
|
if (!ret && req_kernel == VHOST_GET_FEATURES) {
|
|
|
|
/* with tap as the backend, all these features are supported
|
|
|
|
* but not claimed by vhost-net, so we add them back when
|
|
|
|
* reporting to upper layer.
|
|
|
|
*/
|
|
|
|
*((uint64_t *)arg) |= VHOST_KERNEL_GUEST_OFFLOADS_MASK;
|
|
|
|
*((uint64_t *)arg) |= VHOST_KERNEL_HOST_OFFLOADS_MASK;
|
2017-01-13 12:18:40 +00:00
|
|
|
|
|
|
|
/* vhost_kernel will not declare this feature, but it does
|
|
|
|
* support multi-queue.
|
|
|
|
*/
|
|
|
|
if (tap_supporte_mq())
|
|
|
|
*(uint64_t *)arg |= (1ull << VIRTIO_NET_F_MQ);
|
2017-01-13 12:18:39 +00:00
|
|
|
}
|
|
|
|
|
2017-01-13 12:18:38 +00:00
|
|
|
if (vm)
|
|
|
|
free(vm);
|
|
|
|
|
|
|
|
if (ret < 0)
|
|
|
|
PMD_DRV_LOG(ERR, "%s failed: %s",
|
|
|
|
vhost_msg_strings[req], strerror(errno));
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Set up environment to talk with a vhost kernel backend.
|
|
|
|
*
|
|
|
|
* @return
|
|
|
|
* - (-1) if fail to set up;
|
|
|
|
* - (>=0) if successful.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
vhost_kernel_setup(struct virtio_user_dev *dev)
|
|
|
|
{
|
|
|
|
int vhostfd;
|
|
|
|
uint32_t i;
|
|
|
|
|
|
|
|
get_vhost_kernel_max_regions();
|
|
|
|
|
|
|
|
for (i = 0; i < dev->max_queue_pairs; ++i) {
|
|
|
|
vhostfd = open(dev->path, O_RDWR);
|
|
|
|
if (vhostfd < 0) {
|
|
|
|
PMD_DRV_LOG(ERR, "fail to open %s, %s",
|
|
|
|
dev->path, strerror(errno));
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
dev->vhostfds[i] = vhostfd;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
vhost_kernel_set_backend(int vhostfd, int tapfd)
|
|
|
|
{
|
|
|
|
struct vhost_vring_file f;
|
|
|
|
|
|
|
|
f.fd = tapfd;
|
|
|
|
f.index = 0;
|
|
|
|
if (ioctl(vhostfd, VHOST_NET_SET_BACKEND, &f) < 0) {
|
|
|
|
PMD_DRV_LOG(ERR, "VHOST_NET_SET_BACKEND fails, %s",
|
|
|
|
strerror(errno));
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
f.index = 1;
|
|
|
|
if (ioctl(vhostfd, VHOST_NET_SET_BACKEND, &f) < 0) {
|
|
|
|
PMD_DRV_LOG(ERR, "VHOST_NET_SET_BACKEND fails, %s",
|
|
|
|
strerror(errno));
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
vhost_kernel_enable_queue_pair(struct virtio_user_dev *dev,
|
|
|
|
uint16_t pair_idx,
|
|
|
|
int enable)
|
|
|
|
{
|
|
|
|
int hdr_size;
|
|
|
|
int vhostfd;
|
|
|
|
int tapfd;
|
2017-01-13 12:18:40 +00:00
|
|
|
int req_mq = (dev->max_queue_pairs > 1);
|
2017-01-13 12:18:38 +00:00
|
|
|
|
|
|
|
vhostfd = dev->vhostfds[pair_idx];
|
|
|
|
|
|
|
|
if (!enable) {
|
2017-03-13 09:33:21 +00:00
|
|
|
if (dev->tapfds[pair_idx] >= 0) {
|
2017-01-13 12:18:38 +00:00
|
|
|
close(dev->tapfds[pair_idx]);
|
|
|
|
dev->tapfds[pair_idx] = -1;
|
|
|
|
}
|
|
|
|
return vhost_kernel_set_backend(vhostfd, -1);
|
|
|
|
} else if (dev->tapfds[pair_idx] >= 0) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF)) ||
|
|
|
|
(dev->features & (1ULL << VIRTIO_F_VERSION_1)))
|
|
|
|
hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
|
|
|
|
else
|
|
|
|
hdr_size = sizeof(struct virtio_net_hdr);
|
|
|
|
|
2017-12-29 03:38:42 +00:00
|
|
|
tapfd = vhost_kernel_open_tap(&dev->ifname, hdr_size, req_mq,
|
|
|
|
(char *)dev->mac_addr);
|
2017-01-13 12:18:38 +00:00
|
|
|
if (tapfd < 0) {
|
|
|
|
PMD_DRV_LOG(ERR, "fail to open tap for vhost kernel");
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (vhost_kernel_set_backend(vhostfd, tapfd) < 0) {
|
|
|
|
PMD_DRV_LOG(ERR, "fail to set backend for vhost kernel");
|
|
|
|
close(tapfd);
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
dev->tapfds[pair_idx] = tapfd;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct virtio_user_backend_ops ops_kernel = {
|
|
|
|
.setup = vhost_kernel_setup,
|
|
|
|
.send_request = vhost_kernel_ioctl,
|
|
|
|
.enable_qp = vhost_kernel_enable_queue_pair
|
|
|
|
};
|