d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

rdma: Intorduce RDMA providers

Browse Source 
RDMA provider is used as an abstraction layer and allows
to add vendor specific verbs implementation. This patch adds
an interface to create and destroy qpairs and its implementation
using regular verbs API.
Future patches will add functions to queue/flush Work Requests,
enable usage of RDMA providers API in NVMEoF target/initiator and
add Mellanox Direct Verbs implementation. This abstraction layer
is flexible enough to add other vendor's implementations.
The choice of a specific RDMA provider will be made at
the configuration step, it doesn't add any runtime overhead.

Change-Id: If8eec5bd45c82c71f73a14f6e84121cd02a1d65d
Signed-off-by: Alexey Marchuk <alexeymar@mellanox.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/1655
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI
Reviewed-by: Seth Howell <seth.howell@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Jacek Kalwas <jacek.kalwas@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Alexey Marchuk 2020-03-14 10:27:38 +03:00 committed by Tomasz Zawadzki
parent 3ceeab7c8b
commit 7e14f72443
6 changed files with 231 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
include/spdk_internal/rdma.h Normal file
View File

@ -0,0 +1,70 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation. All rights reserved.
* Copyright (c) Mellanox Technologies LTD. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef SPDK_RDMA_H
#define SPDK_RDMA_H
#include <infiniband/verbs.h>
#include <rdma/rdma_cma.h>
#include <rdma/rdma_verbs.h>
struct spdk_rdma_qp_init_attr {
void *qp_context;
struct ibv_cq *send_cq;
struct ibv_cq *recv_cq;
struct ibv_srq *srq;
struct ibv_qp_cap cap;
struct ibv_pd *pd;
};
struct spdk_rdma_qp {
struct ibv_qp *qp;
struct rdma_cm_id *cm_id;
};
/**
* Create RDMA provider specific qpair
* \param cm_id Pointer to RDMACM cm_id
* \param qp_attr Pointer to qpair init attributes
* \return Pointer to a newly created qpair on success or NULL on failure
*/
struct spdk_rdma_qp *spdk_rdma_qp_create(struct rdma_cm_id *cm_id,
struct spdk_rdma_qp_init_attr *qp_attr);
/**
* Destory RDMA provider specific qpair
* \param spdk_rdma_qp Pointer to qpair to be destroyed
*/
void spdk_rdma_qp_destroy(struct spdk_rdma_qp *spdk_rdma_qp);
#endif /* SPDK_RDMA_H */

1
lib/Makefile
View File

@ -48,6 +48,7 @@ DIRS-$(CONFIG_VHOST) += vhost
DIRS-$(CONFIG_VIRTIO) += virtio
DIRS-$(CONFIG_REDUCE) += reduce
DIRS-$(CONFIG_VHOST_INTERNAL_LIB) += rte_vhost
DIRS-$(CONFIG_RDMA) += rdma
# If CONFIG_ENV is pointing at a directory in lib, build it.
# Out-of-tree env implementations must be built separately by the user.

63
lib/rdma/Makefile Normal file
View File

@ -0,0 +1,63 @@
#
# BSD LICENSE
#
# Copyright (c) Intel Corporation. All rights reserved.
# Copyright (c) Mellanox Technologies LTD. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
SO_VER := 1
SO_MINOR := 0
SO_SUFFIX := $(SO_VER).$(SO_MINOR)
SPDK_MAP_FILE = $(abspath $(CURDIR)/spdk_rdma.map)
LIBNAME = rdma
C_SRCS = rdma_verbs.c
LOCAL_SYS_LIBS += -libverbs -lrdmacm
#Attach only if FreeBSD and RDMA is specified with configure
ifeq ($(OS),FreeBSD)
# Mellanox - MLX4 HBA Userspace Library
ifneq ("$(wildcard /usr/lib/libmlx4.*)","")
LOCAL_SYS_LIBS += -lmlx4
endif
# Mellanox - MLX5 HBA Userspace Library
ifneq ("$(wildcard /usr/lib/libmlx5.*)","")
LOCAL_SYS_LIBS += -lmlx5
endif
# Chelsio HBA Userspace Library
ifneq ("$(wildcard /usr/lib/libcxgb4.*)","")
LOCAL_SYS_LIBS += -lcxgb4
endif
endif
include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

87
lib/rdma/rdma_verbs.c Normal file
View File

@ -0,0 +1,87 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation. All rights reserved.
* Copyright (c) Mellanox Technologies LTD. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <rdma/rdma_cma.h>
#include "spdk/stdinc.h"
#include "spdk/string.h"
#include "spdk/likely.h"
#include "spdk_internal/rdma.h"
#include "spdk_internal/log.h"
struct spdk_rdma_qp *
spdk_rdma_qp_create(struct rdma_cm_id *cm_id, struct spdk_rdma_qp_init_attr *qp_attr)
{
struct spdk_rdma_qp *spdk_rdma_qp;
int rc;
struct ibv_qp_init_attr attr = {
.qp_context = qp_attr->qp_context,
.send_cq = qp_attr->send_cq,
.recv_cq = qp_attr->recv_cq,
.srq = qp_attr->srq,
.cap = qp_attr->cap,
.qp_type = IBV_QPT_RC
};
spdk_rdma_qp = calloc(1, sizeof(*spdk_rdma_qp));
if (!spdk_rdma_qp) {
SPDK_ERRLOG("qp memory allocation failed\n");
return NULL;
}
rc = rdma_create_qp(cm_id, qp_attr->pd, &attr);
if (rc) {
SPDK_ERRLOG("Failed to create qp, errno %s (%d)\n", spdk_strerror(errno), errno);
free(spdk_rdma_qp);
return NULL;
}
qp_attr->cap = attr.cap;
spdk_rdma_qp->qp = cm_id->qp;
spdk_rdma_qp->cm_id = cm_id;
return spdk_rdma_qp;
}
void
spdk_rdma_qp_destroy(struct spdk_rdma_qp *spdk_rdma_qp)
{
assert(spdk_rdma_qp != NULL);
if (spdk_rdma_qp->qp) {
rdma_destroy_qp(spdk_rdma_qp->cm_id);
}
free(spdk_rdma_qp);
}

9
lib/rdma/spdk_rdma.map Normal file
View File

@ -0,0 +1,9 @@
{
global:
# Public functions
spdk_rdma_qp_create;
spdk_rdma_qp_destroy;
local: *;
};

1
mk/spdk.lib_deps.mk
View File

@ -55,6 +55,7 @@ DEPDIRS-json := log util
DEPDIRS-nvme := log sock util
DEPDIRS-reduce := log util
DEPDIRS-thread := log util
DEPDIRS-rdma := log util
DEPDIRS-blob := log util thread
DEPDIRS-accel := log util thread