numam-spdk/lib/nvmf/transport.c
Shuhei Matsumoto 063c79d13c nvmf: Optimize nvmf_request_get_buffers by merging buffer and iovec loops
We can merge two loops of req->buffers and req->iov into a single
loop and merge two variables, req->num_buffers and req->iovcnt into
a single variable. For the latter, use req->iovcnt because it is
also used for in-capsule data.

Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: Ia164f2054b98bbcb00308791774e3ffa4fc70baf
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/469489
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Broadcom SPDK FC-NVMe CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Alexey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
2019-09-30 21:11:52 +00:00

493 lines
13 KiB
C

/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation. All rights reserved.
* Copyright (c) 2018-2019 Mellanox Technologies LTD. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "spdk/stdinc.h"
#include "nvmf_internal.h"
#include "transport.h"
#include "spdk/config.h"
#include "spdk/log.h"
#include "spdk/nvmf.h"
#include "spdk/queue.h"
#include "spdk/util.h"
static const struct spdk_nvmf_transport_ops *const g_transport_ops[] = {
#ifdef SPDK_CONFIG_RDMA
&spdk_nvmf_transport_rdma,
#endif
&spdk_nvmf_transport_tcp,
#ifdef SPDK_CONFIG_FC
&spdk_nvmf_transport_fc,
#endif
};
#define NUM_TRANSPORTS (SPDK_COUNTOF(g_transport_ops))
#define MAX_MEMPOOL_NAME_LENGTH 40
static inline const struct spdk_nvmf_transport_ops *
spdk_nvmf_get_transport_ops(enum spdk_nvme_transport_type type)
{
size_t i;
for (i = 0; i != NUM_TRANSPORTS; i++) {
if (g_transport_ops[i]->type == type) {
return g_transport_ops[i];
}
}
return NULL;
}
const struct spdk_nvmf_transport_opts *
spdk_nvmf_get_transport_opts(struct spdk_nvmf_transport *transport)
{
return &transport->opts;
}
spdk_nvme_transport_type_t
spdk_nvmf_get_transport_type(struct spdk_nvmf_transport *transport)
{
return transport->ops->type;
}
struct spdk_nvmf_transport *
spdk_nvmf_transport_create(enum spdk_nvme_transport_type type,
struct spdk_nvmf_transport_opts *opts)
{
const struct spdk_nvmf_transport_ops *ops = NULL;
struct spdk_nvmf_transport *transport;
char spdk_mempool_name[MAX_MEMPOOL_NAME_LENGTH];
int chars_written;
ops = spdk_nvmf_get_transport_ops(type);
if (!ops) {
SPDK_ERRLOG("Transport type '%s' unavailable.\n",
spdk_nvme_transport_id_trtype_str(type));
return NULL;
}
transport = ops->create(opts);
if (!transport) {
SPDK_ERRLOG("Unable to create new transport of type %s\n",
spdk_nvme_transport_id_trtype_str(type));
return NULL;
}
transport->ops = ops;
transport->opts = *opts;
chars_written = snprintf(spdk_mempool_name, MAX_MEMPOOL_NAME_LENGTH, "%s_%s_%s", "spdk_nvmf",
spdk_nvme_transport_id_trtype_str(type), "data");
if (chars_written < 0) {
SPDK_ERRLOG("Unable to generate transport data buffer pool name.\n");
ops->destroy(transport);
return NULL;
}
transport->data_buf_pool = spdk_mempool_create(spdk_mempool_name,
opts->num_shared_buffers,
opts->io_unit_size + NVMF_DATA_BUFFER_ALIGNMENT,
SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
SPDK_ENV_SOCKET_ID_ANY);
if (!transport->data_buf_pool) {
SPDK_ERRLOG("Unable to allocate buffer pool for poll group\n");
ops->destroy(transport);
return NULL;
}
return transport;
}
struct spdk_nvmf_transport *
spdk_nvmf_transport_get_first(struct spdk_nvmf_tgt *tgt)
{
return TAILQ_FIRST(&tgt->transports);
}
struct spdk_nvmf_transport *
spdk_nvmf_transport_get_next(struct spdk_nvmf_transport *transport)
{
return TAILQ_NEXT(transport, link);
}
int
spdk_nvmf_transport_destroy(struct spdk_nvmf_transport *transport)
{
if (transport->data_buf_pool != NULL) {
if (spdk_mempool_count(transport->data_buf_pool) !=
transport->opts.num_shared_buffers) {
SPDK_ERRLOG("transport buffer pool count is %zu but should be %u\n",
spdk_mempool_count(transport->data_buf_pool),
transport->opts.num_shared_buffers);
}
}
spdk_mempool_free(transport->data_buf_pool);
return transport->ops->destroy(transport);
}
int
spdk_nvmf_transport_listen(struct spdk_nvmf_transport *transport,
const struct spdk_nvme_transport_id *trid)
{
return transport->ops->listen(transport, trid);
}
int
spdk_nvmf_transport_stop_listen(struct spdk_nvmf_transport *transport,
const struct spdk_nvme_transport_id *trid)
{
return transport->ops->stop_listen(transport, trid);
}
void
spdk_nvmf_transport_accept(struct spdk_nvmf_transport *transport, new_qpair_fn cb_fn)
{
transport->ops->accept(transport, cb_fn);
}
void
spdk_nvmf_transport_listener_discover(struct spdk_nvmf_transport *transport,
struct spdk_nvme_transport_id *trid,
struct spdk_nvmf_discovery_log_page_entry *entry)
{
transport->ops->listener_discover(transport, trid, entry);
}
struct spdk_nvmf_transport_poll_group *
spdk_nvmf_transport_poll_group_create(struct spdk_nvmf_transport *transport)
{
struct spdk_nvmf_transport_poll_group *group;
struct spdk_nvmf_transport_pg_cache_buf *buf;
group = transport->ops->poll_group_create(transport);
if (!group) {
return NULL;
}
group->transport = transport;
STAILQ_INIT(&group->pending_buf_queue);
STAILQ_INIT(&group->buf_cache);
if (transport->opts.buf_cache_size) {
group->buf_cache_count = 0;
group->buf_cache_size = transport->opts.buf_cache_size;
while (group->buf_cache_count < group->buf_cache_size) {
buf = (struct spdk_nvmf_transport_pg_cache_buf *)spdk_mempool_get(transport->data_buf_pool);
if (!buf) {
SPDK_NOTICELOG("Unable to reserve the full number of buffers for the pg buffer cache.\n");
break;
}
STAILQ_INSERT_HEAD(&group->buf_cache, buf, link);
group->buf_cache_count++;
}
}
return group;
}
struct spdk_nvmf_transport_poll_group *
spdk_nvmf_transport_get_optimal_poll_group(struct spdk_nvmf_transport *transport,
struct spdk_nvmf_qpair *qpair)
{
if (transport->ops->get_optimal_poll_group) {
return transport->ops->get_optimal_poll_group(qpair);
} else {
return NULL;
}
}
void
spdk_nvmf_transport_poll_group_destroy(struct spdk_nvmf_transport_poll_group *group)
{
struct spdk_nvmf_transport_pg_cache_buf *buf, *tmp;
if (!STAILQ_EMPTY(&group->pending_buf_queue)) {
SPDK_ERRLOG("Pending I/O list wasn't empty on poll group destruction\n");
}
STAILQ_FOREACH_SAFE(buf, &group->buf_cache, link, tmp) {
STAILQ_REMOVE(&group->buf_cache, buf, spdk_nvmf_transport_pg_cache_buf, link);
spdk_mempool_put(group->transport->data_buf_pool, buf);
}
group->transport->ops->poll_group_destroy(group);
}
int
spdk_nvmf_transport_poll_group_add(struct spdk_nvmf_transport_poll_group *group,
struct spdk_nvmf_qpair *qpair)
{
if (qpair->transport) {
assert(qpair->transport == group->transport);
if (qpair->transport != group->transport) {
return -1;
}
} else {
qpair->transport = group->transport;
}
return group->transport->ops->poll_group_add(group, qpair);
}
int
spdk_nvmf_transport_poll_group_remove(struct spdk_nvmf_transport_poll_group *group,
struct spdk_nvmf_qpair *qpair)
{
int rc = ENOTSUP;
assert(qpair->transport == group->transport);
if (group->transport->ops->poll_group_remove) {
rc = group->transport->ops->poll_group_remove(group, qpair);
}
return rc;
}
int
spdk_nvmf_transport_poll_group_poll(struct spdk_nvmf_transport_poll_group *group)
{
return group->transport->ops->poll_group_poll(group);
}
int
spdk_nvmf_transport_req_free(struct spdk_nvmf_request *req)
{
return req->qpair->transport->ops->req_free(req);
}
int
spdk_nvmf_transport_req_complete(struct spdk_nvmf_request *req)
{
return req->qpair->transport->ops->req_complete(req);
}
void
spdk_nvmf_transport_qpair_fini(struct spdk_nvmf_qpair *qpair)
{
qpair->transport->ops->qpair_fini(qpair);
}
int
spdk_nvmf_transport_qpair_get_peer_trid(struct spdk_nvmf_qpair *qpair,
struct spdk_nvme_transport_id *trid)
{
return qpair->transport->ops->qpair_get_peer_trid(qpair, trid);
}
int
spdk_nvmf_transport_qpair_get_local_trid(struct spdk_nvmf_qpair *qpair,
struct spdk_nvme_transport_id *trid)
{
return qpair->transport->ops->qpair_get_local_trid(qpair, trid);
}
int
spdk_nvmf_transport_qpair_get_listen_trid(struct spdk_nvmf_qpair *qpair,
struct spdk_nvme_transport_id *trid)
{
return qpair->transport->ops->qpair_get_listen_trid(qpair, trid);
}
bool
spdk_nvmf_transport_opts_init(enum spdk_nvme_transport_type type,
struct spdk_nvmf_transport_opts *opts)
{
const struct spdk_nvmf_transport_ops *ops;
ops = spdk_nvmf_get_transport_ops(type);
if (!ops) {
SPDK_ERRLOG("Transport type %s unavailable.\n",
spdk_nvme_transport_id_trtype_str(type));
return false;
}
ops->opts_init(opts);
return true;
}
int
spdk_nvmf_transport_qpair_set_sqsize(struct spdk_nvmf_qpair *qpair)
{
if (qpair->transport->ops->qpair_set_sqsize) {
return qpair->transport->ops->qpair_set_sqsize(qpair);
}
return 0;
}
int
spdk_nvmf_transport_poll_group_get_stat(struct spdk_nvmf_tgt *tgt,
struct spdk_nvmf_transport *transport,
struct spdk_nvmf_transport_poll_group_stat **stat)
{
if (transport->ops->poll_group_get_stat) {
return transport->ops->poll_group_get_stat(tgt, stat);
} else {
return -ENOTSUP;
}
}
void
spdk_nvmf_transport_poll_group_free_stat(struct spdk_nvmf_transport *transport,
struct spdk_nvmf_transport_poll_group_stat *stat)
{
if (transport->ops->poll_group_free_stat) {
transport->ops->poll_group_free_stat(stat);
}
}
void
spdk_nvmf_request_free_buffers(struct spdk_nvmf_request *req,
struct spdk_nvmf_transport_poll_group *group,
struct spdk_nvmf_transport *transport)
{
uint32_t i;
for (i = 0; i < req->iovcnt; i++) {
if (group->buf_cache_count < group->buf_cache_size) {
STAILQ_INSERT_HEAD(&group->buf_cache,
(struct spdk_nvmf_transport_pg_cache_buf *)req->buffers[i],
link);
group->buf_cache_count++;
} else {
spdk_mempool_put(transport->data_buf_pool, req->buffers[i]);
}
req->iov[i].iov_base = NULL;
req->buffers[i] = NULL;
req->iov[i].iov_len = 0;
}
req->data_from_pool = false;
}
static inline int
nvmf_request_set_buffer(struct spdk_nvmf_request *req, void *buf, uint32_t length,
uint32_t io_unit_size)
{
req->buffers[req->iovcnt] = buf;
req->iov[req->iovcnt].iov_base = (void *)((uintptr_t)(buf + NVMF_DATA_BUFFER_MASK) &
~NVMF_DATA_BUFFER_MASK);
req->iov[req->iovcnt].iov_len = spdk_min(length, io_unit_size);
length -= req->iov[req->iovcnt].iov_len;
req->iovcnt++;
return length;
}
static int
nvmf_request_get_buffers(struct spdk_nvmf_request *req,
struct spdk_nvmf_transport_poll_group *group,
struct spdk_nvmf_transport *transport,
uint32_t length)
{
uint32_t io_unit_size = transport->opts.io_unit_size;
uint32_t num_buffers;
uint32_t i = 0, j;
void *buffer, *buffers[NVMF_REQ_MAX_BUFFERS];
/* If the number of buffers is too large, then we know the I/O is larger than allowed.
* Fail it.
*/
num_buffers = SPDK_CEIL_DIV(length, io_unit_size);
if (num_buffers + req->iovcnt > NVMF_REQ_MAX_BUFFERS) {
return -EINVAL;
}
while (i < num_buffers) {
if (!(STAILQ_EMPTY(&group->buf_cache))) {
group->buf_cache_count--;
buffer = STAILQ_FIRST(&group->buf_cache);
STAILQ_REMOVE_HEAD(&group->buf_cache, link);
assert(buffer != NULL);
length = nvmf_request_set_buffer(req, buffer, length, io_unit_size);
i++;
} else {
if (spdk_mempool_get_bulk(transport->data_buf_pool, buffers,
num_buffers - i)) {
return -ENOMEM;
}
for (j = 0; j < num_buffers - i; j++) {
length = nvmf_request_set_buffer(req, buffers[j], length, io_unit_size);
}
i += num_buffers - i;
}
}
assert(length == 0);
req->data_from_pool = true;
return 0;
}
int
spdk_nvmf_request_get_buffers(struct spdk_nvmf_request *req,
struct spdk_nvmf_transport_poll_group *group,
struct spdk_nvmf_transport *transport,
uint32_t length)
{
int rc;
req->iovcnt = 0;
rc = nvmf_request_get_buffers(req, group, transport, length);
if (rc == -ENOMEM) {
spdk_nvmf_request_free_buffers(req, group, transport);
}
return rc;
}
int
spdk_nvmf_request_get_buffers_multi(struct spdk_nvmf_request *req,
struct spdk_nvmf_transport_poll_group *group,
struct spdk_nvmf_transport *transport,
uint32_t *lengths, uint32_t num_lengths)
{
int rc = 0;
uint32_t i;
req->iovcnt = 0;
for (i = 0; i < num_lengths; i++) {
rc = nvmf_request_get_buffers(req, group, transport, lengths[i]);
if (rc != 0) {
goto err_exit;
}
}
return 0;
err_exit:
spdk_nvmf_request_free_buffers(req, group, transport);
return rc;
}