vfio: support VF token
The Linux kernel module vfio-pci introduces the VF token to enable SR-IOV support since 5.7. The VF token can be set by a vfio-pci based PF driver and must be known by the vfio-pci based VF driver in order to gain access to the device. Since the vfio-pci module uses the VF token as internal data to provide the collaboration between SR-IOV PF and VFs, so DPDK can use the same VF token for all PF devices by specifying the related EAL option. Signed-off-by: Haiyue Wang <haiyue.wang@intel.com> Acked-by: Anatoly Burakov <anatoly.burakov@intel.com> Acked-by: Andrew Rybchenko <arybchenko@solarflare.com> Tested-by: Harman Kalra <hkalra@marvell.com>
This commit is contained in:
parent
edca6d883e
commit
598be72395
@ -78,11 +78,44 @@ Note that in order to use VFIO, your kernel must support it.
|
||||
VFIO kernel modules have been included in the Linux kernel since version 3.6.0 and are usually present by default,
|
||||
however please consult your distributions documentation to make sure that is the case.
|
||||
|
||||
The ``vfio-pci`` module since Linux version 5.7 supports the creation of virtual
|
||||
functions. After the PF is bound to vfio-pci module, the user can create the VFs
|
||||
by sysfs interface, and these VFs are bound to vfio-pci module automatically.
|
||||
|
||||
When the PF is bound to vfio-pci, it has initial VF token generated by random. For
|
||||
security reason, this token is write only, the user can't read it from the kernel
|
||||
directly. To access the VF, the user needs to start the PF with token parameter to
|
||||
setup a VF token in UUID format, then the VF can be accessed with this new token.
|
||||
|
||||
Since the ``vfio-pci`` module uses the VF token as internal data to provide the
|
||||
collaboration between SR-IOV PF and VFs, so DPDK can use the same VF token for all
|
||||
PF devices which bound to one application. This VF token can be specified by the EAL
|
||||
parameter ``--vfio-vf-token``.
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
1. Generate the VF token by uuid command
|
||||
14d63f20-8445-11ea-8900-1f9ce7d5650d
|
||||
|
||||
2. sudo modprobe vfio-pci enable_sriov=1
|
||||
|
||||
2. ./usertools/dpdk-devbind.py -b vfio-pci 0000:86:00.0
|
||||
|
||||
3. echo 2 > /sys/bus/pci/devices/0000:86:00.0/sriov_numvfs
|
||||
|
||||
4. Start the PF:
|
||||
./x86_64-native-linux-gcc/app/testpmd -l 22-25 -n 4 -w 86:00.0 \
|
||||
--vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=pf -- -i
|
||||
|
||||
5. Start the VF:
|
||||
./x86_64-native-linux-gcc/app/testpmd -l 26-29 -n 4 -w 86:02.0 \
|
||||
--vfio-vf-token=14d63f20-8445-11ea-8900-1f9ce7d5650d --file-prefix=vf0 -- -i
|
||||
|
||||
Also, to use VFIO, both kernel and BIOS must support and be configured to use IO virtualization (such as Intel® VT-d).
|
||||
|
||||
.. note::
|
||||
|
||||
``vfio-pci`` module doesn't support the creation of virtual functions.
|
||||
``vfio-pci`` module doesn't support the creation of virtual functions before Linux version 5.7.
|
||||
|
||||
For proper operation of VFIO when running DPDK applications as a non-privileged user, correct permissions should also be set up.
|
||||
This can be done by using the DPDK setup script (called dpdk-setup.sh and located in the usertools directory).
|
||||
|
@ -40,6 +40,10 @@ Device-related options
|
||||
|
||||
Use specified interrupt mode for devices bound to VFIO kernel driver.
|
||||
|
||||
* ``--vfio-vf-token <uuid>``
|
||||
|
||||
Use specified VF token for devices bound to VFIO kernel driver.
|
||||
|
||||
Multiprocessing-related options
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
@ -56,6 +56,13 @@ New Features
|
||||
Also, make sure to start the actual text at the margin.
|
||||
=========================================================
|
||||
|
||||
* **Added the support for vfio-pci new VF token interface.**
|
||||
|
||||
From Linux 5.7, vfio-pci supports to bind both SR-IOV PF and the created VFs,
|
||||
it uses a shared VF token (UUID) to represent the collaboration between PF
|
||||
and VFs. Update DPDK PCI driver to gain the access to the PF and VFs devices
|
||||
by appending the VF token parameter.
|
||||
|
||||
* **Added the RegEx Library, a generic RegEx service library.**
|
||||
|
||||
Added the RegEx library which provides an API for offload of regular
|
||||
|
@ -95,6 +95,7 @@ eal_long_options[] = {
|
||||
{OPT_SYSLOG, 1, NULL, OPT_SYSLOG_NUM },
|
||||
{OPT_VDEV, 1, NULL, OPT_VDEV_NUM },
|
||||
{OPT_VFIO_INTR, 1, NULL, OPT_VFIO_INTR_NUM },
|
||||
{OPT_VFIO_VF_TOKEN, 1, NULL, OPT_VFIO_VF_TOKEN_NUM },
|
||||
{OPT_VMWARE_TSC_MAP, 0, NULL, OPT_VMWARE_TSC_MAP_NUM },
|
||||
{OPT_LEGACY_MEM, 0, NULL, OPT_LEGACY_MEM_NUM },
|
||||
{OPT_SINGLE_FILE_SEGMENTS, 0, NULL, OPT_SINGLE_FILE_SEGMENTS_NUM},
|
||||
@ -328,6 +329,8 @@ eal_reset_internal_config(struct internal_config *internal_cfg)
|
||||
|
||||
/* if set to NONE, interrupt mode is determined automatically */
|
||||
internal_cfg->vfio_intr_mode = RTE_INTR_MODE_NONE;
|
||||
memset(internal_cfg->vfio_vf_token, 0,
|
||||
sizeof(internal_cfg->vfio_vf_token));
|
||||
|
||||
#ifdef RTE_LIBEAL_USE_HPET
|
||||
internal_cfg->no_hpet = 0;
|
||||
|
@ -72,6 +72,8 @@ struct internal_config {
|
||||
volatile int syslog_facility; /**< facility passed to openlog() */
|
||||
/** default interrupt mode for VFIO */
|
||||
volatile enum rte_intr_mode vfio_intr_mode;
|
||||
/** the shared VF token for VFIO-PCI bound PF and VFs devices */
|
||||
rte_uuid_t vfio_vf_token;
|
||||
char *hugefile_prefix; /**< the base filename of hugetlbfs files */
|
||||
char *hugepage_dir; /**< specific hugetlbfs directory to use */
|
||||
char *user_mbuf_pool_ops_name;
|
||||
|
@ -69,6 +69,8 @@ enum {
|
||||
OPT_VDEV_NUM,
|
||||
#define OPT_VFIO_INTR "vfio-intr"
|
||||
OPT_VFIO_INTR_NUM,
|
||||
#define OPT_VFIO_VF_TOKEN "vfio-vf-token"
|
||||
OPT_VFIO_VF_TOKEN_NUM,
|
||||
#define OPT_VMWARE_TSC_MAP "vmware-tsc-map"
|
||||
OPT_VMWARE_TSC_MAP_NUM,
|
||||
#define OPT_LEGACY_MEM "legacy-mem"
|
||||
|
@ -966,6 +966,11 @@ rte_eal_vfio_intr_mode(void)
|
||||
return RTE_INTR_MODE_NONE;
|
||||
}
|
||||
|
||||
void
|
||||
rte_eal_vfio_get_vf_token(__rte_unused rte_uuid_t vf_token)
|
||||
{
|
||||
}
|
||||
|
||||
int rte_vfio_setup_device(__rte_unused const char *sysfs_base,
|
||||
__rte_unused const char *dev_addr,
|
||||
__rte_unused int *vfio_dev_fd,
|
||||
|
@ -19,6 +19,7 @@
|
||||
#include <rte_compat.h>
|
||||
#include <rte_per_lcore.h>
|
||||
#include <rte_bus.h>
|
||||
#include <rte_uuid.h>
|
||||
|
||||
#include <rte_pci_dev_feature_defs.h>
|
||||
|
||||
@ -438,6 +439,19 @@ int rte_eal_create_uio_dev(void);
|
||||
*/
|
||||
enum rte_intr_mode rte_eal_vfio_intr_mode(void);
|
||||
|
||||
/**
|
||||
* @warning
|
||||
* @b EXPERIMENTAL: this API may change without prior notice
|
||||
*
|
||||
* Copy the user-configured vfio VF token.
|
||||
*
|
||||
* @param vf_token
|
||||
* vfio VF token configured with the command line is copied
|
||||
* into this parameter, zero uuid by default.
|
||||
*/
|
||||
__rte_experimental
|
||||
void rte_eal_vfio_get_vf_token(rte_uuid_t vf_token);
|
||||
|
||||
/**
|
||||
* A wrap API for syscall gettid.
|
||||
*
|
||||
|
@ -540,6 +540,7 @@ eal_usage(const char *prgname)
|
||||
" --"OPT_FILE_PREFIX" Prefix for hugepage filenames\n"
|
||||
" --"OPT_CREATE_UIO_DEV" Create /dev/uioX (usually done by hotplug)\n"
|
||||
" --"OPT_VFIO_INTR" Interrupt mode for VFIO (legacy|msi|msix)\n"
|
||||
" --"OPT_VFIO_VF_TOKEN" VF token (UUID) shared between SR-IOV PF and VFs\n"
|
||||
" --"OPT_LEGACY_MEM" Legacy memory mode (no dynamic allocation, contiguous segments)\n"
|
||||
" --"OPT_SINGLE_FILE_SEGMENTS" Put all hugepage memory in single files\n"
|
||||
" --"OPT_MATCH_ALLOCATIONS" Free hugepages exactly as allocated\n"
|
||||
@ -620,6 +621,20 @@ eal_parse_vfio_intr(const char *mode)
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int
|
||||
eal_parse_vfio_vf_token(const char *vf_token)
|
||||
{
|
||||
struct internal_config *cfg = eal_get_internal_configuration();
|
||||
rte_uuid_t uuid;
|
||||
|
||||
if (!rte_uuid_parse(vf_token, uuid)) {
|
||||
rte_uuid_copy(cfg->vfio_vf_token, uuid);
|
||||
return 0;
|
||||
}
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Parse the arguments for --log-level only */
|
||||
static void
|
||||
eal_log_level_parse(int argc, char **argv)
|
||||
@ -762,6 +777,16 @@ eal_parse_args(int argc, char **argv)
|
||||
}
|
||||
break;
|
||||
|
||||
case OPT_VFIO_VF_TOKEN_NUM:
|
||||
if (eal_parse_vfio_vf_token(optarg) < 0) {
|
||||
RTE_LOG(ERR, EAL, "invalid parameters for --"
|
||||
OPT_VFIO_VF_TOKEN "\n");
|
||||
eal_usage(prgname);
|
||||
ret = -1;
|
||||
goto out;
|
||||
}
|
||||
break;
|
||||
|
||||
case OPT_CREATE_UIO_DEV_NUM:
|
||||
internal_conf->create_uio_dev = 1;
|
||||
break;
|
||||
@ -1342,6 +1367,14 @@ rte_eal_vfio_intr_mode(void)
|
||||
return internal_conf->vfio_intr_mode;
|
||||
}
|
||||
|
||||
void
|
||||
rte_eal_vfio_get_vf_token(rte_uuid_t vf_token)
|
||||
{
|
||||
struct internal_config *cfg = eal_get_internal_configuration();
|
||||
|
||||
rte_uuid_copy(vf_token, cfg->vfio_vf_token);
|
||||
}
|
||||
|
||||
int
|
||||
rte_eal_check_module(const char *module_name)
|
||||
{
|
||||
|
@ -714,6 +714,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
|
||||
int vfio_container_fd;
|
||||
int vfio_group_fd;
|
||||
int iommu_group_num;
|
||||
rte_uuid_t vf_token;
|
||||
int i, ret;
|
||||
const struct internal_config *internal_conf =
|
||||
eal_get_internal_configuration();
|
||||
@ -899,6 +900,23 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
|
||||
t->type_id, t->name);
|
||||
}
|
||||
|
||||
rte_eal_vfio_get_vf_token(vf_token);
|
||||
|
||||
/* get a file descriptor for the device with VF token firstly */
|
||||
if (!rte_uuid_is_null(vf_token)) {
|
||||
char vf_token_str[RTE_UUID_STRLEN];
|
||||
char dev[PATH_MAX];
|
||||
|
||||
rte_uuid_unparse(vf_token, vf_token_str, sizeof(vf_token_str));
|
||||
snprintf(dev, sizeof(dev),
|
||||
"%s vf_token=%s", dev_addr, vf_token_str);
|
||||
|
||||
*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD,
|
||||
dev);
|
||||
if (*vfio_dev_fd >= 0)
|
||||
goto dev_get_info;
|
||||
}
|
||||
|
||||
/* get a file descriptor for the device */
|
||||
*vfio_dev_fd = ioctl(vfio_group_fd, VFIO_GROUP_GET_DEVICE_FD, dev_addr);
|
||||
if (*vfio_dev_fd < 0) {
|
||||
@ -914,6 +932,7 @@ rte_vfio_setup_device(const char *sysfs_base, const char *dev_addr,
|
||||
}
|
||||
|
||||
/* test and setup the device */
|
||||
dev_get_info:
|
||||
ret = ioctl(*vfio_dev_fd, VFIO_DEVICE_GET_INFO, device_info);
|
||||
if (ret) {
|
||||
RTE_LOG(ERR, EAL, " %s cannot get device info, "
|
||||
|
@ -386,6 +386,9 @@ EXPERIMENTAL {
|
||||
rte_trace_point_lookup;
|
||||
rte_trace_regexp;
|
||||
rte_trace_save;
|
||||
|
||||
# added in 20.08
|
||||
rte_eal_vfio_get_vf_token;
|
||||
};
|
||||
|
||||
INTERNAL {
|
||||
|
Loading…
x
Reference in New Issue
Block a user