d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

env: Check supported iommu address width before using iova-mode=va

Browse Source 
DPDK by default guesses that it should be using iova-mode=va
so that it can support running as an unprivileged user. However,
some systems (especially virtual machines) don't have an IOMMU capable
of handling the full virtual address space and DPDK doesn't
currently catch that. Add a check in SPDK and force iova-mode=pa
here.

Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/475149 (master)

(cherry picked from commit 97b0f7733f)
Change-Id: Ib3a5691a584190feaab4b9064b5a500e361328f2
Signed-off-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/478349
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
This commit is contained in:
Ben Walker 2019-11-19 11:28:14 -07:00 committed by Tomasz Zawadzki
parent 848927d96a
commit 1b0b8d04f8
2 changed files with 84 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/spdk/stdinc.h
View File

@ -57,6 +57,7 @@ extern "C" {
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <time.h>
/* POSIX */

84
lib/env_dpdk/init.c
View File

@ -175,6 +175,75 @@ spdk_push_arg(char *args[], int *argcount, char *arg)
return tmp;
}
#if defined(__linux__) && defined(__x86_64__)
/* TODO: Can likely get this value from rlimits in the future */
#define SPDK_IOMMU_VA_REQUIRED_WIDTH 48
#define VTD_CAP_MGAW_SHIFT 16
#define VTD_CAP_MGAW_MASK (0x3F << VTD_CAP_MGAW_SHIFT)
static int
spdk_get_iommu_width(void)
{
DIR *dir;
FILE *file;
struct dirent *entry;
char mgaw_path[64];
char buf[64];
char *end;
long long int val;
int width, tmp;
dir = opendir("/sys/devices/virtual/iommu/");
if (dir == NULL) {
return -EINVAL;
}
width = 0;
while ((entry = readdir(dir)) != NULL) {
/* Find directories named "dmar0", "dmar1", etc */
if (strncmp(entry->d_name, "dmar", sizeof("dmar") - 1) != 0) {
continue;
}
tmp = snprintf(mgaw_path, sizeof(mgaw_path), "/sys/devices/virtual/iommu/%s/intel-iommu/cap",
entry->d_name);
if ((unsigned)tmp >= sizeof(mgaw_path)) {
continue;
}
file = fopen(mgaw_path, "r");
if (file == NULL) {
continue;
}
if (fgets(buf, sizeof(buf), file) == NULL) {
fclose(file);
continue;
}
val = strtoll(buf, &end, 16);
if (val == LLONG_MIN || val == LLONG_MAX) {
fclose(file);
continue;
}
tmp = ((val & VTD_CAP_MGAW_MASK) >> VTD_CAP_MGAW_SHIFT) + 1;
if (width == 0 || tmp < width) {
width = tmp;
}
fclose(file);
}
closedir(dir);
return width;
}
#endif
static int
spdk_build_eal_cmdline(const struct spdk_env_opts *opts)
{
@ -337,7 +406,19 @@ spdk_build_eal_cmdline(const struct spdk_env_opts *opts)
#ifdef __linux__
#ifdef __PPC64__
#if defined(__x86_64__)
/* DPDK by default guesses that it should be using iova-mode=va so that it can
* support running as an unprivileged user. However, some systems (especially
* virtual machines) don't have an IOMMU capable of handling the full virtual
* address space and DPDK doesn't currently catch that. Add a check in SPDK
* and force iova-mode=pa here. */
if (spdk_get_iommu_width() < SPDK_IOMMU_VA_REQUIRED_WIDTH) {
args = spdk_push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa"));
if (args == NULL) {
return -1;
}
}
#elif defined(__PPC64__)
/* On Linux + PowerPC, DPDK doesn't support VA mode at all. Unfortunately, it doesn't correctly
* auto-detect at the moment, so we'll just force it here. */
args = spdk_push_arg(args, &argcount, _sprintf_alloc("--iova-mode=pa"));
@ -346,6 +427,7 @@ spdk_build_eal_cmdline(const struct spdk_env_opts *opts)
}
#endif
/* Set the base virtual address - it must be an address that is not in the
* ASAN shadow region, otherwise ASAN-enabled builds will ignore the
* mmap hint.