vhost: enable VFIO
Vhost needs to register memory given by guest in VFIO container to be able to do any DMA using this memory. Currently DPDK doesn't provide any interface to handle guest memory, so for now lets find container fd in /proc/self/fd/ directory and provide some VFIO internal API that finally should extend DPDK API. Change-Id: Iee9d496367ccd61219068fc0eadc17e786ff0731 Signed-off-by: Pawel Wodkowski <pawelx.wodkowski@intel.com>
This commit is contained in:
Pawel Wodkowski
Jim Harris
parent
4613ed760c
commit
9aaccfe3d7
@ -149,11 +149,6 @@ assigned to vm.
|
||||
|
||||
# Known bugs and limitations {#vhost_bugs}
|
||||
|
||||
## VFIO driver is not supported
|
||||
Currently, VFIO driver is not supported by vhost library. Please use UIO
|
||||
driver if running SPDK vhost app. Otherwise any IO to physical device
|
||||
(eg. IOAT or NVMe) will fail. Support for vhost with VFIO is in progress.
|
||||
|
||||
## Hot plug is not supported
|
||||
Hot plug is not supported in vhost yet. Event queue path doesn't handle that
|
||||
case. While hot plug will be just ignored, hot removal might cause segmentation
|
||||
|
||||
@ -38,7 +38,7 @@ CFLAGS += -I.
|
||||
CFLAGS += -Irte_vhost
|
||||
CFLAGS += $(ENV_CFLAGS)
|
||||
|
||||
C_SRCS = task.c vhost.c vhost_rpc.c
|
||||
C_SRCS = task.c vhost.c vhost_rpc.c vhost_iommu.c
|
||||
|
||||
LIBNAME = vhost
|
||||
|
||||
|
||||
@ -57,6 +57,7 @@
|
||||
|
||||
#include "spdk/vhost.h"
|
||||
#include "task.h"
|
||||
#include "vhost_iommu.h"
|
||||
|
||||
static uint32_t g_num_ctrlrs[RTE_MAX_LCORE];
|
||||
|
||||
@ -72,6 +73,9 @@ static char dev_dirname[PATH_MAX] = "";
|
||||
struct spdk_vaddr_region {
|
||||
void *vaddr;
|
||||
uint64_t len;
|
||||
|
||||
uint64_t host_user_addr;
|
||||
uint64_t host_user_size;
|
||||
};
|
||||
|
||||
/*
|
||||
@ -647,6 +651,8 @@ add_vdev_cb(void *arg1, void *arg2)
|
||||
{
|
||||
struct spdk_vhost_scsi_ctrlr *vdev = arg1;
|
||||
struct virtio_memory_region *region;
|
||||
struct spdk_vaddr_region *vregion;
|
||||
uint64_t start, end, len;
|
||||
uint32_t i;
|
||||
|
||||
for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
|
||||
@ -658,16 +664,20 @@ add_vdev_cb(void *arg1, void *arg2)
|
||||
SPDK_NOTICELOG("Started poller for vhost controller %s on lcore %d\n", vdev->name, vdev->lcore);
|
||||
vdev->nregions = vdev->dev->mem->nregions;
|
||||
for (i = 0; i < vdev->nregions; i++) {
|
||||
uint64_t start, end, len;
|
||||
region = &vdev->dev->mem->regions[i];
|
||||
start = FLOOR_2MB(region->mmap_addr);
|
||||
end = CEIL_2MB(region->mmap_addr + region->mmap_size);
|
||||
len = end - start;
|
||||
vdev->region[i].vaddr = (void *)start;
|
||||
vdev->region[i].len = len;
|
||||
SPDK_NOTICELOG("Registering VM memory for vtophys translation - 0x%jx len:0x%jx\n",
|
||||
start, len);
|
||||
spdk_mem_register(vdev->region[i].vaddr, vdev->region[i].len);
|
||||
vregion = &vdev->region[i];
|
||||
vregion->vaddr = (void *)start;
|
||||
vregion->len = len;
|
||||
vregion->host_user_addr = region->host_user_addr;
|
||||
vregion->host_user_size = region->size;
|
||||
|
||||
SPDK_NOTICELOG("Registering VM memory for vtophys translation - %p len:0x%jx\n",
|
||||
vdev->region[i].vaddr, vdev->region[i].len);
|
||||
spdk_mem_register(vregion->vaddr, vregion->len);
|
||||
spdk_iommu_mem_register(vregion->host_user_addr, vregion->host_user_size);
|
||||
}
|
||||
|
||||
spdk_poller_register(&vdev->requestq_poller, vdev_worker, vdev, vdev->lcore, 0);
|
||||
@ -680,6 +690,7 @@ static void
|
||||
remove_vdev_cb(void *arg1, void *arg2)
|
||||
{
|
||||
struct spdk_vhost_scsi_ctrlr *vdev = arg1;
|
||||
struct spdk_vaddr_region *reg;
|
||||
uint32_t i;
|
||||
|
||||
for (i = 0; i < SPDK_VHOST_SCSI_CTRLR_MAX_DEVS; i++) {
|
||||
@ -691,7 +702,9 @@ remove_vdev_cb(void *arg1, void *arg2)
|
||||
|
||||
SPDK_NOTICELOG("Stopping poller for vhost controller %s\n", vdev->name);
|
||||
for (i = 0; i < vdev->nregions; i++) {
|
||||
spdk_mem_unregister(vdev->region[i].vaddr, vdev->region[i].len);
|
||||
reg = &vdev->region[i];
|
||||
spdk_iommu_mem_unregister(reg->host_user_addr, reg->host_user_size);
|
||||
spdk_mem_unregister(reg->vaddr, reg->len);
|
||||
}
|
||||
|
||||
vdev->nregions = 0;
|
||||
@ -1123,28 +1136,6 @@ session_start(void *arg)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if /sys/kernel/iommu_groups exists and is not empty
|
||||
*/
|
||||
static bool
|
||||
has_iommu_groups(void)
|
||||
{
|
||||
struct dirent *d;
|
||||
int iommu_count = 0;
|
||||
DIR *dir = opendir("/sys/kernel/iommu_groups");
|
||||
|
||||
if (dir == NULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
while (iommu_count < 3 && (d = readdir(dir)) != NULL) {
|
||||
++iommu_count;
|
||||
}
|
||||
|
||||
closedir(dir);
|
||||
return iommu_count > 2; /* there will always be ./ and ../ entries */
|
||||
}
|
||||
|
||||
void
|
||||
spdk_vhost_startup(void *arg1, void *arg2)
|
||||
{
|
||||
@ -1168,15 +1159,6 @@ spdk_vhost_startup(void *arg1, void *arg2)
|
||||
if (ret != 0)
|
||||
rte_exit(EXIT_FAILURE, "Cannot construct vhost controllers\n");
|
||||
|
||||
if (has_iommu_groups()) {
|
||||
RTE_LOG(WARNING, VHOST_CONFIG,
|
||||
"Currently VFIO driver is not supported by vhost library\n"
|
||||
"Although guest might be able to boot and see devices, it will not be able\n"
|
||||
"to do any IO to physical devices (Malloc with IOAT, NVMe).\n"
|
||||
"Please use uio_pci_generic driver with vhost app/library.\n"
|
||||
"See documentation for more information.\n");
|
||||
}
|
||||
|
||||
rte_vhost_driver_callback_register(&virtio_net_device_ops);
|
||||
|
||||
if (pthread_create(&tid, NULL, &session_start, NULL) < 0)
|
||||
|
||||
234
lib/vhost/vhost_iommu.c
Normal file
234
lib/vhost/vhost_iommu.c
Normal file
@ -0,0 +1,234 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright(c) Intel Corporation. All rights reserved.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include <inttypes.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <dirent.h>
|
||||
#include <errno.h>
|
||||
#include <assert.h>
|
||||
|
||||
#include <sys/ioctl.h>
|
||||
#include <linux/vfio.h>
|
||||
|
||||
#include <spdk/env.h>
|
||||
#include <spdk/util.h>
|
||||
#include "spdk_internal/log.h"
|
||||
#include "vhost_iommu.h"
|
||||
|
||||
static struct {
|
||||
int need_init;
|
||||
int container_fd;
|
||||
} vfio_cfg = { 1, -1 };
|
||||
|
||||
/* Internal DPDK function forward declaration */
|
||||
int pci_vfio_is_enabled(void);
|
||||
|
||||
/* Discover DPDK vfio container fd. This is to be removed if DPDK API
|
||||
* provides interface for memory registration in VFIO container.
|
||||
*
|
||||
* Return -1 on error, 0 on success (VFIO is used or not)
|
||||
*/
|
||||
static int
|
||||
vfio_cfg_init(void)
|
||||
{
|
||||
char proc_fd_path[PATH_MAX + 1];
|
||||
char link_path[PATH_MAX + 1];
|
||||
const char vfio_path[] = "/dev/vfio/vfio";
|
||||
const int vfio_path_len = sizeof(vfio_path) - 1;
|
||||
DIR *dir;
|
||||
struct dirent *d;
|
||||
|
||||
if (!vfio_cfg.need_init) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
vfio_cfg.need_init = 0;
|
||||
if (!pci_vfio_is_enabled()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
dir = opendir("/proc/self/fd");
|
||||
while ((d = readdir(dir)) != NULL) {
|
||||
if (d->d_type != DT_LNK)
|
||||
continue;
|
||||
|
||||
snprintf(proc_fd_path, sizeof(proc_fd_path), "/proc/self/fd/%s", d->d_name);
|
||||
if (readlink(proc_fd_path, link_path, sizeof(link_path)) != vfio_path_len)
|
||||
continue;
|
||||
|
||||
if (memcmp(link_path, vfio_path, vfio_path_len) == 0) {
|
||||
sscanf(d->d_name, "%d", &vfio_cfg.container_fd);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
closedir(dir);
|
||||
|
||||
if (vfio_cfg.container_fd < 0) {
|
||||
SPDK_ERRLOG("Failed to discover DPDK VFIO container fd.\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
vfio_pci_memory_region_map(int vfio_container_fd, uint64_t vaddr, uint64_t phys_addr, uint64_t size)
|
||||
{
|
||||
struct vfio_iommu_type1_dma_map dma_map;
|
||||
int ret;
|
||||
|
||||
dma_map.argsz = sizeof(dma_map);
|
||||
dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
|
||||
dma_map.vaddr = vaddr;
|
||||
dma_map.iova = phys_addr;
|
||||
dma_map.size = size;
|
||||
|
||||
SPDK_TRACELOG(SPDK_TRACE_VHOST_VFIO, "MAP vaddr:%p phys:%p len:%#"PRIx64"\n", (void *)vaddr,
|
||||
(void *)phys_addr, size);
|
||||
ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map);
|
||||
|
||||
if (ret) {
|
||||
SPDK_ERRLOG("Cannot set up DMA mapping, error %d (%s)\n", errno, strerror(errno));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int
|
||||
vfio_pci_memory_region_unmap(int vfio_container_fd, uint64_t phys_addr, uint64_t size)
|
||||
{
|
||||
struct vfio_iommu_type1_dma_unmap dma_unmap;
|
||||
int ret;
|
||||
|
||||
dma_unmap.argsz = sizeof(dma_unmap);
|
||||
dma_unmap.flags = 0;
|
||||
dma_unmap.iova = phys_addr;
|
||||
dma_unmap.size = size;
|
||||
|
||||
SPDK_TRACELOG(SPDK_TRACE_VHOST_VFIO, "UNMAP phys:%p len:%#"PRIx64"\n", (void *)phys_addr, size);
|
||||
ret = ioctl(vfio_container_fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap);
|
||||
|
||||
if (ret) {
|
||||
SPDK_ERRLOG("Cannot clear DMA mapping, error %d (%s)\n", errno, strerror(errno));
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int
|
||||
vfio_pci_memory_region_op(uint64_t vaddr, uint64_t phys_addr, uint64_t size, int op)
|
||||
{
|
||||
|
||||
if (vfio_cfg.container_fd == -1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (op == VFIO_IOMMU_MAP_DMA) {
|
||||
return vfio_pci_memory_region_map(vfio_cfg.container_fd, vaddr, phys_addr, size);
|
||||
} else {
|
||||
return vfio_pci_memory_region_unmap(vfio_cfg.container_fd, phys_addr, size);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#define SHIFT_2MB 21 /* (1 << 21) == 2MB */
|
||||
#define MASK_2MB ((1ULL << SHIFT_2MB) - 1)
|
||||
|
||||
static int
|
||||
spdk_vfio_mem_op(uint64_t addr, uint64_t len, int dma_op)
|
||||
{
|
||||
const uint64_t len_2mb = 1 << SHIFT_2MB;
|
||||
uint64_t vaddr, vend, phaddr, phend, vlen;
|
||||
int ret = 0;
|
||||
|
||||
if (vfio_cfg_init() != 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (vfio_cfg.container_fd == -1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
vaddr = addr;
|
||||
while (len > 0) {
|
||||
vlen = spdk_min(len_2mb - (vaddr & MASK_2MB), len);
|
||||
vend = vaddr + vlen;
|
||||
|
||||
phaddr = spdk_vtophys((void *)vaddr);
|
||||
phend = spdk_vtophys((void *)(vend - 1));
|
||||
|
||||
if (phaddr == SPDK_VTOPHYS_ERROR || phend == SPDK_VTOPHYS_ERROR ||
|
||||
phend - phaddr > vlen - 1) {
|
||||
SPDK_ERRLOG("Invalid memory region addr: %p len:%"PRIu64" "
|
||||
"spdk_vtophys(%p) = %p spdk_vtophys(%p) = %p\n",
|
||||
(void *)addr, len, (void *)vaddr, (void *)phaddr,
|
||||
(void *)vend, (void *)phend);
|
||||
ret = -1;
|
||||
break;
|
||||
}
|
||||
|
||||
ret = vfio_pci_memory_region_op(vaddr, phaddr, vlen, dma_op);
|
||||
if (ret) {
|
||||
SPDK_ERRLOG("Failed to %s region region vaddr=%p phys_addr=%p len=%#"PRIx64"\n",
|
||||
(dma_op == VFIO_IOMMU_MAP_DMA ? "map" : "unmap"), (void *)vaddr,
|
||||
(void *)phaddr, vlen);
|
||||
break;
|
||||
}
|
||||
|
||||
vaddr += vlen;
|
||||
len -= vlen;
|
||||
|
||||
assert(len == 0 || (vaddr & MASK_2MB) == 0);
|
||||
}
|
||||
|
||||
if (ret) {
|
||||
spdk_vfio_mem_op(addr, vaddr - addr, VFIO_IOMMU_UNMAP_DMA);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int spdk_iommu_mem_register(uint64_t addr, uint64_t len)
|
||||
{
|
||||
return spdk_vfio_mem_op(addr, len, VFIO_IOMMU_MAP_DMA);
|
||||
}
|
||||
|
||||
int spdk_iommu_mem_unregister(uint64_t addr, uint64_t len)
|
||||
{
|
||||
return spdk_vfio_mem_op(addr, len, VFIO_IOMMU_UNMAP_DMA);
|
||||
}
|
||||
|
||||
SPDK_LOG_REGISTER_TRACE_FLAG("vhost_vfio", SPDK_TRACE_VHOST_VFIO)
|
||||
61
lib/vhost/vhost_iommu.h
Normal file
61
lib/vhost/vhost_iommu.h
Normal file
@ -0,0 +1,61 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright (c) Intel Corporation.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef SPDK_VHOST_IOMMU_H
|
||||
#define SPDK_VHOST_IOMMU_H
|
||||
|
||||
#include <inttypes.h>
|
||||
|
||||
/**
|
||||
* Register given memory block in currently used IOMMU. If no IOMMU is used this
|
||||
* function do nothing but still should be called.
|
||||
*
|
||||
* \param addr Start of memory block
|
||||
* \param len Length of memory block.
|
||||
* \return 0 on success, -1 on error.
|
||||
*/
|
||||
int spdk_iommu_mem_register(uint64_t addr, uint64_t len);
|
||||
|
||||
/**
|
||||
* Unregister previously registered memory block in currently used IOMMU. If no
|
||||
* IOMMU is used this function do nothing but still should be called.
|
||||
*
|
||||
* \note This functiom might fail for invalid memory block.
|
||||
*
|
||||
* \param addr Start of memory block
|
||||
* \param len Length of memory block.
|
||||
* \return 0 on success, -1 on error.
|
||||
*/
|
||||
int spdk_iommu_mem_unregister(uint64_t addr, uint64_t len);
|
||||
|
||||
#endif /* SPDK_VHOST_IOMMU_H */
|
||||
Loading…
Reference in New Issue
Block a user