d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvmf: Remove controller.[ch] and probe for each subsystem.

Browse Source 
This is a much simpler approach and is only slightly
less efficient.

Change-Id: I909de376d576a74156c1be447e90e7dbc240f025
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Ben Walker 2016-07-21 11:24:17 -07:00 committed by Benjamin Walker
parent fd3f7ee3b3
commit 944695ca21
9 changed files with 77 additions and 460 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
etc/spdk/nvmf.conf.in
View File

@ -33,20 +33,6 @@
# Set the maximum number of outstanding I/O per queue.
#MaxQueueDepth 128
# NVMe Device Whitelist
# Users may specify which NVMe devices to claim by their PCI
# domain, bus, device, and function. The format is dddd:bb:dd.f, which is
# the same format displayed by lspci or in /sys/bus/pci/devices. The second
# argument is a "name" for the device that can be anything. The name
# is referenced later in the Subsystem section.
#
# Alternatively, the user can specify ClaimAllDevices. All
# NVMe devices will be claimed and named Nvme0, Nvme1, etc.
[Nvme]
BDF 0000:00:00.0 Nvme0
BDF 0000:01:00.0 Nvme1
# ClaimAllDevices Yes
# Define an NVMf Subsystem.
# - NQN is required and must be unique.
# - Mode may be either "Direct" or "Virtual". Direct means that physical
@ -62,13 +48,15 @@
# - Between 0 and 255 Host directives are allowed. This defines the
# NQNs of allowed hosts. If no Host directive is specified, all hosts
# are allowed to connect.
# - Exactly 1 Controller directive.
# - Exactly 1 NVMe directive specifying an NVMe device by PCI BDF. The
# PCI domain:bus:device.function can be replaced by "*" to indicate
# any PCI device.
[Subsystem1]
NQN nqn.2016-06.io.spdk:cnode1
Mode Direct
Listen RDMA 15.15.15.2:4420
Host nqn.2016-06.io.spdk:init
Controller Nvme0
NVMe 0000:00:00.0
# Multiple subsystems are allowed.
[Subsystem2]
@ -76,5 +64,5 @@
Mode Direct
Listen RDMA 192.168.2.21:4420
Host nqn.2016-06.io.spdk:init
Controller Nvme1
NVMe 0000:01:00.0

4
lib/nvmf/Makefile
View File

@ -36,8 +36,8 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
CFLAGS += $(DPDK_INC)
LIBNAME = nvmf
C_SRCS = controller.c subsystem.c conf.c \
nvmf.c request.c session.c transport.c
C_SRCS = subsystem.c conf.c nvmf.c \
request.c session.c transport.c
C_SRCS-$(CONFIG_RDMA) += rdma.c

161
lib/nvmf/conf.c
View File

@ -40,7 +40,6 @@
#include <rte_lcore.h>
#include "conf.h"
#include "controller.h"
#include "nvmf_internal.h"
#include "subsystem.h"
#include "transport.h"
@ -52,6 +51,15 @@
#define PORTNUMSTRLEN 32
struct spdk_nvmf_probe_ctx {
struct spdk_nvmf_subsystem *subsystem;
bool any;
int domain;
int bus;
int device;
int function;
};
static int
spdk_nvmf_parse_nvmf_tgt(void)
{
@ -182,82 +190,46 @@ spdk_nvmf_parse_addr(char *listen_addr, char **host, char **port)
return 0;
}
static int
spdk_nvmf_parse_nvme(void)
static bool
probe_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr_opts *opts)
{
struct spdk_conf_section *sp;
struct nvme_bdf_whitelist *whitelist = NULL;
const char *val;
bool claim_all = false;
bool unbind_from_kernel = false;
int i = 0;
struct spdk_nvmf_probe_ctx *ctx = cb_ctx;
uint16_t found_domain = spdk_pci_device_get_domain(dev);
uint8_t found_bus = spdk_pci_device_get_bus(dev);
uint8_t found_dev = spdk_pci_device_get_dev(dev);
uint8_t found_func = spdk_pci_device_get_func(dev);
SPDK_NOTICELOG("Probing device %x:%x:%x.%x\n",
found_domain, found_bus, found_dev, found_func);
if (ctx->any) {
return true;
}
if (found_domain == ctx->domain &&
found_bus == ctx->bus &&
found_dev == ctx->device &&
found_func == ctx->function) {
if (!spdk_pci_device_has_non_uio_driver(dev)) {
return true;
}
SPDK_ERRLOG("Requested device is still bound to the kernel. Unbind your NVMe devices first.\n");
}
return false;
}
static void
attach_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr *ctrlr,
const struct spdk_nvme_ctrlr_opts *opts)
{
struct spdk_nvmf_probe_ctx *ctx = cb_ctx;
int rc;
sp = spdk_conf_find_section(NULL, "Nvme");
if (sp == NULL) {
SPDK_ERRLOG("NVMe device section in config file not found!\n");
return -1;
rc = nvmf_subsystem_add_ctrlr(ctx->subsystem, ctrlr);
if (rc < 0) {
SPDK_ERRLOG("Failed to add controller to subsystem\n");
}
val = spdk_conf_section_get_val(sp, "ClaimAllDevices");
if (val != NULL) {
if (!strcmp(val, "Yes")) {
claim_all = true;
}
}
val = spdk_conf_section_get_val(sp, "UnbindFromKernel");
if (val != NULL) {
if (!strcmp(val, "Yes")) {
unbind_from_kernel = true;
}
}
if (!claim_all) {
for (i = 0; ; i++) {
unsigned int domain, bus, dev, func;
val = spdk_conf_section_get_nmval(sp, "BDF", i, 0);
if (val == NULL) {
break;
}
whitelist = realloc(whitelist, sizeof(*whitelist) * (i + 1));
rc = sscanf(val, "%x:%x:%x.%x", &domain, &bus, &dev, &func);
if (rc != 4) {
SPDK_ERRLOG("Invalid format for BDF: %s\n", val);
free(whitelist);
return -1;
}
whitelist[i].domain = domain;
whitelist[i].bus = bus;
whitelist[i].dev = dev;
whitelist[i].func = func;
val = spdk_conf_section_get_nmval(sp, "BDF", i, 1);
if (val == NULL) {
SPDK_ERRLOG("BDF section with no device name\n");
free(whitelist);
return -1;
}
snprintf(whitelist[i].name, MAX_NVME_NAME_LENGTH, "%s", val);
}
if (i == 0) {
SPDK_ERRLOG("No BDF section\n");
return -1;
}
}
rc = spdk_nvmf_init_nvme(whitelist, i,
claim_all, unbind_from_kernel);
free(whitelist);
return rc;
}
static int
@ -310,9 +282,9 @@ spdk_nvmf_allocate_lcore(uint64_t mask, uint32_t lcore)
static int
spdk_nvmf_parse_subsystem(struct spdk_conf_section *sp)
{
const char *val, *nqn, *mode;
const char *nqn, *mode, *bdf;
struct spdk_nvmf_subsystem *subsystem;
struct spdk_nvmf_ctrlr *nvmf_ctrlr;
struct spdk_nvmf_probe_ctx ctx = { 0 };
int i, ret;
uint64_t mask;
uint32_t lcore;
@ -400,31 +372,30 @@ spdk_nvmf_parse_subsystem(struct spdk_conf_section *sp)
spdk_nvmf_subsystem_add_host(subsystem, host_nqn);
}
val = spdk_conf_section_get_val(sp, "Controller");
if (val == NULL) {
SPDK_ERRLOG("Subsystem %d: missing Controller\n", sp->num);
/* Parse NVMe section */
bdf = spdk_conf_section_get_val(sp, "NVMe");
if (bdf == NULL) {
SPDK_ERRLOG("Subsystem %d: missing NVMe directive\n", sp->num);
nvmf_delete_subsystem(subsystem);
return -1;
}
/* claim this controller from the available controller list */
nvmf_ctrlr = spdk_nvmf_ctrlr_claim(val);
if (nvmf_ctrlr == NULL) {
SPDK_ERRLOG("Subsystem %d: NVMe controller %s not found\n", sp->num, val);
nvmf_delete_subsystem(subsystem);
return -1;
ctx.subsystem = subsystem;
if (strcmp(bdf, "*") == 0) {
ctx.any = true;
} else {
ret = sscanf(bdf, "%x:%x:%x.%x", &ctx.domain, &ctx.bus, &ctx.device, &ctx.function);
if (ret != 4) {
SPDK_ERRLOG("Invalid format for NVMe BDF: %s\n", bdf);
return -1;
}
ctx.any = false;
}
ret = nvmf_subsystem_add_ctrlr(subsystem, nvmf_ctrlr->ctrlr);
if (ret < 0) {
SPDK_ERRLOG("Subsystem %d: adding controller %s failed\n", sp->num, val);
nvmf_delete_subsystem(subsystem);
return -1;
if (spdk_nvme_probe(&ctx, probe_cb, attach_cb, NULL)) {
SPDK_ERRLOG("One or more controllers failed in spdk_nvme_probe()\n");
}
SPDK_TRACELOG(SPDK_TRACE_DEBUG, " NVMf Subsystem: Nvme Controller: %s , %p\n",
nvmf_ctrlr->name, nvmf_ctrlr->ctrlr);
return 0;
}
@ -458,12 +429,6 @@ spdk_nvmf_parse_conf(void)
return rc;
}
/* NVMe sections */
rc = spdk_nvmf_parse_nvme();
if (rc < 0) {
return rc;
}
/* Subsystem sections */
rc = spdk_nvmf_parse_subsystems();
if (rc < 0) {

269
lib/nvmf/controller.c
View File

@ -1,269 +0,0 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "controller.h"
#include "spdk/conf.h"
#include "spdk/nvme.h"
#include "spdk/log.h"
#include "spdk/trace.h"
static TAILQ_HEAD(, spdk_nvmf_ctrlr) g_ctrlrs = TAILQ_HEAD_INITIALIZER(g_ctrlrs);
#define SPDK_NVMF_MAX_NVME_DEVICES 64
struct spdk_nvmf_probe_ctx {
bool claim_all;
bool unbind_from_kernel;
int whitelist_count;
struct nvme_bdf_whitelist *whitelist;
};
static void
spdk_nvmf_complete_ctrlr_aer(struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_cpl *cpl)
{
/* TODO: Temporarily disabled during refactoring. */
#if 0
struct spdk_nvmf_subsystem *subsystem;
struct nvmf_session *sess;
int i;
/*
* Scan the whitelist for any subsystems claiming namespaces
* associated with this NVMe controller.
*/
for (i = 0; i < g_num_nvme_devices; i++) {
if (g_whitelist[i].ctrlr == ctrlr &&
g_whitelist[i].subsystem != NULL) {
subsystem = g_whitelist[i].subsystem;
TAILQ_FOREACH(sess, &subsystem->sessions, entries) {
if (sess->aer_req == NULL) {
continue;
}
SPDK_TRACELOG(SPDK_TRACE_NVMF, "Process session AER request, sess %p, req %p\n",
sess, sess->aer_req);
nvmf_complete_cmd(sess->aer_req, cpl);
/* clear this AER from the session */
sess->aer_req = NULL;
}
}
}
#endif
}
static void
aer_cb(void *arg, const struct spdk_nvme_cpl *cpl)
{
struct spdk_nvme_ctrlr *ctrlr = arg;
if (spdk_nvme_cpl_is_error(cpl)) {
fprintf(stderr, "Nvme AER failed!\n");
return;
}
SPDK_TRACELOG(SPDK_TRACE_NVMF, " Nvme AER callback, log_page_id %x\n",
(cpl->cdw0 & 0xFF0000) >> 16);
spdk_nvmf_complete_ctrlr_aer(ctrlr, cpl);
}
static void
spdk_nvmf_ctrlr_create(char *name, int domain, int bus, int dev, int func,
struct spdk_nvme_ctrlr *ctrlr)
{
struct spdk_nvmf_ctrlr *nvmf_ctrlr;
nvmf_ctrlr = calloc(1, sizeof(struct spdk_nvmf_ctrlr));
if (nvmf_ctrlr == NULL) {
SPDK_ERRLOG("allocate nvmf_ctrlr failed.\n");
return;
}
SPDK_TRACELOG(SPDK_TRACE_NVMF, "Found physical NVMe device. Name: %s\n", name);
nvmf_ctrlr->ctrlr = ctrlr;
snprintf(nvmf_ctrlr->name, MAX_NVME_NAME_LENGTH, "%s", name);
spdk_nvme_ctrlr_register_aer_callback(ctrlr, aer_cb, ctrlr);
TAILQ_INSERT_HEAD(&g_ctrlrs, nvmf_ctrlr, entry);
}
static bool
probe_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr_opts *opts)
{
struct spdk_nvmf_probe_ctx *ctx = cb_ctx;
uint16_t found_domain = spdk_pci_device_get_domain(dev);
uint8_t found_bus = spdk_pci_device_get_bus(dev);
uint8_t found_dev = spdk_pci_device_get_dev(dev);
uint8_t found_func = spdk_pci_device_get_func(dev);
int i;
bool claim_device = false;
SPDK_NOTICELOG("Probing device %x:%x:%x.%x\n",
found_domain, found_bus, found_dev, found_func);
if (ctx->claim_all) {
claim_device = true;
} else {
for (i = 0; i < SPDK_NVMF_MAX_NVME_DEVICES; i++) {
if (found_domain == ctx->whitelist[i].domain &&
found_bus == ctx->whitelist[i].bus &&
found_dev == ctx->whitelist[i].dev &&
found_func == ctx->whitelist[i].func) {
claim_device = true;
break;
}
}
}
if (!claim_device) {
return false;
}
if (spdk_pci_device_has_non_uio_driver(dev)) {
if (ctx->unbind_from_kernel) {
if (spdk_pci_device_switch_to_uio_driver(dev) == 0) {
return true;
}
}
} else {
return true;
}
return false;
}
static void
attach_cb(void *cb_ctx, struct spdk_pci_device *dev, struct spdk_nvme_ctrlr *ctrlr,
const struct spdk_nvme_ctrlr_opts *opts)
{
struct spdk_nvmf_probe_ctx *ctx = cb_ctx;
uint16_t found_domain = spdk_pci_device_get_domain(dev);
uint8_t found_bus = spdk_pci_device_get_bus(dev);
uint8_t found_dev = spdk_pci_device_get_dev(dev);
uint8_t found_func = spdk_pci_device_get_func(dev);
int i;
SPDK_NOTICELOG("Attempting to claim device %x:%x:%x.%x\n",
found_domain, found_bus, found_dev, found_func);
if (ctx->claim_all) {
/* If claim_all is true, whitelist_count can be repurposed here safely */
char name[64];
snprintf(name, 64, "Nvme%d", ctx->whitelist_count);
spdk_nvmf_ctrlr_create(name, found_domain, found_bus,
found_dev, found_func, ctrlr);
ctx->whitelist_count++;
return;
}
for (i = 0; i < SPDK_NVMF_MAX_NVME_DEVICES; i++) {
if (found_domain == ctx->whitelist[i].domain &&
found_bus == ctx->whitelist[i].bus &&
found_dev == ctx->whitelist[i].dev &&
found_func == ctx->whitelist[i].func) {
spdk_nvmf_ctrlr_create(ctx->whitelist[i].name, found_domain, found_bus,
found_dev, found_func, ctrlr);
return;
}
}
}
int
spdk_nvmf_init_nvme(struct nvme_bdf_whitelist *whitelist, size_t whitelist_count,
bool claim_all, bool unbind_from_kernel)
{
struct spdk_nvmf_probe_ctx ctx = { 0 };
ctx.whitelist = whitelist;
ctx.whitelist_count = whitelist_count;
ctx.claim_all = claim_all;
ctx.unbind_from_kernel = unbind_from_kernel;
/* Probe the physical NVMe devices */
if (spdk_nvme_probe(&ctx, probe_cb, attach_cb, NULL)) {
SPDK_ERRLOG("One or more controllers failed in spdk_nvme_probe()\n");
}
/* check whether any nvme controller is probed */
if (TAILQ_EMPTY(&g_ctrlrs)) {
SPDK_ERRLOG("No nvme controllers are probed\n");
return -1;
}
return 0;
}
int
spdk_nvmf_shutdown_nvme(void)
{
struct spdk_nvmf_ctrlr *ctrlr, *tctrlr;
TAILQ_FOREACH_SAFE(ctrlr, &g_ctrlrs, entry, tctrlr) {
TAILQ_REMOVE(&g_ctrlrs, ctrlr, entry);
spdk_nvme_detach(ctrlr->ctrlr);
free(ctrlr);
}
return 0;
}
struct spdk_nvmf_ctrlr *
spdk_nvmf_ctrlr_claim(const char *name)
{
struct spdk_nvmf_ctrlr *ctrlr, *tctrlr;
if (name == NULL) {
return NULL;
}
SPDK_TRACELOG(SPDK_TRACE_NVMF, "Attempting to claim NVMe controller %s\n", name);
TAILQ_FOREACH_SAFE(ctrlr, &g_ctrlrs, entry, tctrlr) {
if (strncmp(ctrlr->name, name, MAX_NVME_NAME_LENGTH) == 0) {
if (ctrlr->claimed) {
SPDK_ERRLOG("Two subsystems are attempting to claim the same NVMe controller.\n");
return NULL;
}
ctrlr->claimed = true;
return ctrlr;
}
}
return NULL;
}

66
lib/nvmf/controller.h
View File

@ -1,66 +0,0 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef NVMF_CONTROLLER_H
#define NVMF_CONTROLLER_H
#include <stdbool.h>
#include "nvmf_internal.h"
#define MAX_NVME_NAME_LENGTH 64
struct nvme_bdf_whitelist {
uint16_t domain;
uint8_t bus;
uint8_t dev;
uint8_t func;
char name[MAX_NVME_NAME_LENGTH];
};
struct spdk_nvmf_ctrlr {
struct spdk_nvme_ctrlr *ctrlr;
char name[MAX_NVME_NAME_LENGTH];
bool claimed;
TAILQ_ENTRY(spdk_nvmf_ctrlr) entry;
};
int spdk_nvmf_init_nvme(struct nvme_bdf_whitelist *whitelist, size_t whitelist_count,
bool claim_all, bool unbind_from_kernel);
int spdk_nvmf_shutdown_nvme(void);
struct spdk_nvmf_ctrlr *
spdk_nvmf_ctrlr_claim(const char *name);
#endif

2
lib/nvmf/nvmf.c
View File

@ -40,7 +40,6 @@
#include "spdk/log.h"
#include "spdk/conf.h"
#include "conf.h"
#include "controller.h"
#include "subsystem.h"
#include "transport.h"
#include "spdk/trace.h"
@ -193,7 +192,6 @@ static int
nvmf_tgt_subsystem_fini(void)
{
spdk_shutdown_nvmf_subsystems();
spdk_nvmf_shutdown_nvme();
spdk_nvmf_transport_fini();
pthread_mutex_destroy(&g_nvmf_tgt.mutex);

5
lib/nvmf/subsystem.c
View File

@ -33,7 +33,6 @@
#include <ctype.h>
#include "controller.h"
#include "nvmf_internal.h"
#include "session.h"
#include "subsystem.h"
@ -154,6 +153,10 @@ nvmf_delete_subsystem(struct spdk_nvmf_subsystem *subsystem)
spdk_nvmf_session_destruct(subsystem->session);
}
if (subsystem->ctrlr) {
spdk_nvme_detach(subsystem->ctrlr);
}
TAILQ_REMOVE(&g_subsystems, subsystem, entries);
free(subsystem);

2
lib/nvmf/subsystem.h
View File

@ -34,6 +34,8 @@
#ifndef SPDK_NVMF_SUBSYSTEM_H
#define SPDK_NVMF_SUBSYSTEM_H
#include "nvmf_internal.h"
#include "spdk/event.h"
#include "spdk/nvme.h"
#include "spdk/queue.h"

6
test/nvmf/nvmf.conf
View File

@ -5,12 +5,8 @@
[Nvmf]
MaxQueuesPerSession 4
[Nvme]
ClaimAllDevices Yes
UnbindFromKernel Yes
[Subsystem1]
NQN "nqn.2016-06.io.spdk:cnode1"
Mode Direct
Listen RDMA 192.168.100.8:4420
Controller Nvme0
NVMe *