a18c0acd8c
Increase the size of small and large buffers in the bdev layer to store DIF. Increase is the amount necessary to store 16 byte metadata to the block formatted with 512 + 16. 512 + 16 is the current maximum ratio of metadata per block. This change will be done in the iSCSI library in the next patch and other libraries may be done later. Hence add and use an new macro SPDK_BDEV_BUF_SIZE_WITH_MD(x) for convenience. Change-Id: I4b5498f56c9baf3e3ed93dd1c757998d7ce65141 Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com> Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/444558 Tested-by: SPDK CI Jenkins <sys_sgci@intel.com> Reviewed-by: Changpeng Liu <changpeng.liu@intel.com> Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com> Reviewed-by: Jim Harris <james.r.harris@intel.com>
4546 lines
123 KiB
C
4546 lines
123 KiB
C
/*-
|
|
* BSD LICENSE
|
|
*
|
|
* Copyright (c) Intel Corporation.
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
*
|
|
* * Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* * Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in
|
|
* the documentation and/or other materials provided with the
|
|
* distribution.
|
|
* * Neither the name of Intel Corporation nor the names of its
|
|
* contributors may be used to endorse or promote products derived
|
|
* from this software without specific prior written permission.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
#include "spdk/stdinc.h"
|
|
|
|
#include "spdk/bdev.h"
|
|
#include "spdk/conf.h"
|
|
|
|
#include "spdk/config.h"
|
|
#include "spdk/env.h"
|
|
#include "spdk/event.h"
|
|
#include "spdk/thread.h"
|
|
#include "spdk/likely.h"
|
|
#include "spdk/queue.h"
|
|
#include "spdk/nvme_spec.h"
|
|
#include "spdk/scsi_spec.h"
|
|
#include "spdk/util.h"
|
|
#include "spdk/trace.h"
|
|
|
|
#include "spdk/bdev_module.h"
|
|
#include "spdk_internal/log.h"
|
|
#include "spdk/string.h"
|
|
|
|
#ifdef SPDK_CONFIG_VTUNE
|
|
#include "ittnotify.h"
|
|
#include "ittnotify_types.h"
|
|
int __itt_init_ittlib(const char *, __itt_group_id);
|
|
#endif
|
|
|
|
#define SPDK_BDEV_IO_POOL_SIZE (64 * 1024)
|
|
#define SPDK_BDEV_IO_CACHE_SIZE 256
|
|
#define BUF_SMALL_POOL_SIZE 8192
|
|
#define BUF_LARGE_POOL_SIZE 1024
|
|
#define NOMEM_THRESHOLD_COUNT 8
|
|
#define ZERO_BUFFER_SIZE 0x100000
|
|
|
|
#define OWNER_BDEV 0x2
|
|
|
|
#define OBJECT_BDEV_IO 0x2
|
|
|
|
#define TRACE_GROUP_BDEV 0x3
|
|
#define TRACE_BDEV_IO_START SPDK_TPOINT_ID(TRACE_GROUP_BDEV, 0x0)
|
|
#define TRACE_BDEV_IO_DONE SPDK_TPOINT_ID(TRACE_GROUP_BDEV, 0x1)
|
|
|
|
#define SPDK_BDEV_QOS_TIMESLICE_IN_USEC 1000
|
|
#define SPDK_BDEV_QOS_MIN_IO_PER_TIMESLICE 1
|
|
#define SPDK_BDEV_QOS_MIN_BYTE_PER_TIMESLICE 512
|
|
#define SPDK_BDEV_QOS_MIN_IOS_PER_SEC 10000
|
|
#define SPDK_BDEV_QOS_MIN_BYTES_PER_SEC (10 * 1024 * 1024)
|
|
#define SPDK_BDEV_QOS_LIMIT_NOT_DEFINED UINT64_MAX
|
|
|
|
#define SPDK_BDEV_POOL_ALIGNMENT 512
|
|
|
|
static const char *qos_conf_type[] = {"Limit_IOPS",
|
|
"Limit_BPS", "Limit_Read_BPS", "Limit_Write_BPS"
|
|
};
|
|
static const char *qos_rpc_type[] = {"rw_ios_per_sec",
|
|
"rw_mbytes_per_sec", "r_mbytes_per_sec", "w_mbytes_per_sec"
|
|
};
|
|
|
|
TAILQ_HEAD(spdk_bdev_list, spdk_bdev);
|
|
|
|
struct spdk_bdev_mgr {
|
|
struct spdk_mempool *bdev_io_pool;
|
|
|
|
struct spdk_mempool *buf_small_pool;
|
|
struct spdk_mempool *buf_large_pool;
|
|
|
|
void *zero_buffer;
|
|
|
|
TAILQ_HEAD(bdev_module_list, spdk_bdev_module) bdev_modules;
|
|
|
|
struct spdk_bdev_list bdevs;
|
|
|
|
bool init_complete;
|
|
bool module_init_complete;
|
|
|
|
#ifdef SPDK_CONFIG_VTUNE
|
|
__itt_domain *domain;
|
|
#endif
|
|
};
|
|
|
|
static struct spdk_bdev_mgr g_bdev_mgr = {
|
|
.bdev_modules = TAILQ_HEAD_INITIALIZER(g_bdev_mgr.bdev_modules),
|
|
.bdevs = TAILQ_HEAD_INITIALIZER(g_bdev_mgr.bdevs),
|
|
.init_complete = false,
|
|
.module_init_complete = false,
|
|
};
|
|
|
|
static struct spdk_bdev_opts g_bdev_opts = {
|
|
.bdev_io_pool_size = SPDK_BDEV_IO_POOL_SIZE,
|
|
.bdev_io_cache_size = SPDK_BDEV_IO_CACHE_SIZE,
|
|
};
|
|
|
|
static spdk_bdev_init_cb g_init_cb_fn = NULL;
|
|
static void *g_init_cb_arg = NULL;
|
|
|
|
static spdk_bdev_fini_cb g_fini_cb_fn = NULL;
|
|
static void *g_fini_cb_arg = NULL;
|
|
static struct spdk_thread *g_fini_thread = NULL;
|
|
|
|
struct spdk_bdev_qos_limit {
|
|
/** IOs or bytes allowed per second (i.e., 1s). */
|
|
uint64_t limit;
|
|
|
|
/** Remaining IOs or bytes allowed in current timeslice (e.g., 1ms).
|
|
* For remaining bytes, allowed to run negative if an I/O is submitted when
|
|
* some bytes are remaining, but the I/O is bigger than that amount. The
|
|
* excess will be deducted from the next timeslice.
|
|
*/
|
|
int64_t remaining_this_timeslice;
|
|
|
|
/** Minimum allowed IOs or bytes to be issued in one timeslice (e.g., 1ms). */
|
|
uint32_t min_per_timeslice;
|
|
|
|
/** Maximum allowed IOs or bytes to be issued in one timeslice (e.g., 1ms). */
|
|
uint32_t max_per_timeslice;
|
|
|
|
/** Function to check whether to queue the IO. */
|
|
bool (*queue_io)(const struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io);
|
|
|
|
/** Function to update for the submitted IO. */
|
|
void (*update_quota)(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io);
|
|
};
|
|
|
|
struct spdk_bdev_qos {
|
|
/** Types of structure of rate limits. */
|
|
struct spdk_bdev_qos_limit rate_limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES];
|
|
|
|
/** The channel that all I/O are funneled through. */
|
|
struct spdk_bdev_channel *ch;
|
|
|
|
/** The thread on which the poller is running. */
|
|
struct spdk_thread *thread;
|
|
|
|
/** Queue of I/O waiting to be issued. */
|
|
bdev_io_tailq_t queued;
|
|
|
|
/** Size of a timeslice in tsc ticks. */
|
|
uint64_t timeslice_size;
|
|
|
|
/** Timestamp of start of last timeslice. */
|
|
uint64_t last_timeslice;
|
|
|
|
/** Poller that processes queued I/O commands each time slice. */
|
|
struct spdk_poller *poller;
|
|
};
|
|
|
|
struct spdk_bdev_mgmt_channel {
|
|
bdev_io_stailq_t need_buf_small;
|
|
bdev_io_stailq_t need_buf_large;
|
|
|
|
/*
|
|
* Each thread keeps a cache of bdev_io - this allows
|
|
* bdev threads which are *not* DPDK threads to still
|
|
* benefit from a per-thread bdev_io cache. Without
|
|
* this, non-DPDK threads fetching from the mempool
|
|
* incur a cmpxchg on get and put.
|
|
*/
|
|
bdev_io_stailq_t per_thread_cache;
|
|
uint32_t per_thread_cache_count;
|
|
uint32_t bdev_io_cache_size;
|
|
|
|
TAILQ_HEAD(, spdk_bdev_shared_resource) shared_resources;
|
|
TAILQ_HEAD(, spdk_bdev_io_wait_entry) io_wait_queue;
|
|
};
|
|
|
|
/*
|
|
* Per-module (or per-io_device) data. Multiple bdevs built on the same io_device
|
|
* will queue here their IO that awaits retry. It makes it possible to retry sending
|
|
* IO to one bdev after IO from other bdev completes.
|
|
*/
|
|
struct spdk_bdev_shared_resource {
|
|
/* The bdev management channel */
|
|
struct spdk_bdev_mgmt_channel *mgmt_ch;
|
|
|
|
/*
|
|
* Count of I/O submitted to bdev module and waiting for completion.
|
|
* Incremented before submit_request() is called on an spdk_bdev_io.
|
|
*/
|
|
uint64_t io_outstanding;
|
|
|
|
/*
|
|
* Queue of IO awaiting retry because of a previous NOMEM status returned
|
|
* on this channel.
|
|
*/
|
|
bdev_io_tailq_t nomem_io;
|
|
|
|
/*
|
|
* Threshold which io_outstanding must drop to before retrying nomem_io.
|
|
*/
|
|
uint64_t nomem_threshold;
|
|
|
|
/* I/O channel allocated by a bdev module */
|
|
struct spdk_io_channel *shared_ch;
|
|
|
|
/* Refcount of bdev channels using this resource */
|
|
uint32_t ref;
|
|
|
|
TAILQ_ENTRY(spdk_bdev_shared_resource) link;
|
|
};
|
|
|
|
#define BDEV_CH_RESET_IN_PROGRESS (1 << 0)
|
|
#define BDEV_CH_QOS_ENABLED (1 << 1)
|
|
|
|
struct spdk_bdev_channel {
|
|
struct spdk_bdev *bdev;
|
|
|
|
/* The channel for the underlying device */
|
|
struct spdk_io_channel *channel;
|
|
|
|
/* Per io_device per thread data */
|
|
struct spdk_bdev_shared_resource *shared_resource;
|
|
|
|
struct spdk_bdev_io_stat stat;
|
|
|
|
/*
|
|
* Count of I/O submitted through this channel and waiting for completion.
|
|
* Incremented before submit_request() is called on an spdk_bdev_io.
|
|
*/
|
|
uint64_t io_outstanding;
|
|
|
|
bdev_io_tailq_t queued_resets;
|
|
|
|
uint32_t flags;
|
|
|
|
struct spdk_histogram_data *histogram;
|
|
|
|
#ifdef SPDK_CONFIG_VTUNE
|
|
uint64_t start_tsc;
|
|
uint64_t interval_tsc;
|
|
__itt_string_handle *handle;
|
|
struct spdk_bdev_io_stat prev_stat;
|
|
#endif
|
|
|
|
};
|
|
|
|
struct spdk_bdev_desc {
|
|
struct spdk_bdev *bdev;
|
|
struct spdk_thread *thread;
|
|
spdk_bdev_remove_cb_t remove_cb;
|
|
void *remove_ctx;
|
|
bool remove_scheduled;
|
|
bool closed;
|
|
bool write;
|
|
TAILQ_ENTRY(spdk_bdev_desc) link;
|
|
};
|
|
|
|
struct spdk_bdev_iostat_ctx {
|
|
struct spdk_bdev_io_stat *stat;
|
|
spdk_bdev_get_device_stat_cb cb;
|
|
void *cb_arg;
|
|
};
|
|
|
|
struct set_qos_limit_ctx {
|
|
void (*cb_fn)(void *cb_arg, int status);
|
|
void *cb_arg;
|
|
struct spdk_bdev *bdev;
|
|
};
|
|
|
|
#define __bdev_to_io_dev(bdev) (((char *)bdev) + 1)
|
|
#define __bdev_from_io_dev(io_dev) ((struct spdk_bdev *)(((char *)io_dev) - 1))
|
|
|
|
static void _spdk_bdev_write_zero_buffer_done(struct spdk_bdev_io *bdev_io, bool success,
|
|
void *cb_arg);
|
|
static void _spdk_bdev_write_zero_buffer_next(void *_bdev_io);
|
|
|
|
static void _spdk_bdev_enable_qos_msg(struct spdk_io_channel_iter *i);
|
|
static void _spdk_bdev_enable_qos_done(struct spdk_io_channel_iter *i, int status);
|
|
|
|
void
|
|
spdk_bdev_get_opts(struct spdk_bdev_opts *opts)
|
|
{
|
|
*opts = g_bdev_opts;
|
|
}
|
|
|
|
int
|
|
spdk_bdev_set_opts(struct spdk_bdev_opts *opts)
|
|
{
|
|
uint32_t min_pool_size;
|
|
|
|
/*
|
|
* Add 1 to the thread count to account for the extra mgmt_ch that gets created during subsystem
|
|
* initialization. A second mgmt_ch will be created on the same thread when the application starts
|
|
* but before the deferred put_io_channel event is executed for the first mgmt_ch.
|
|
*/
|
|
min_pool_size = opts->bdev_io_cache_size * (spdk_thread_get_count() + 1);
|
|
if (opts->bdev_io_pool_size < min_pool_size) {
|
|
SPDK_ERRLOG("bdev_io_pool_size %" PRIu32 " is not compatible with bdev_io_cache_size %" PRIu32
|
|
" and %" PRIu32 " threads\n", opts->bdev_io_pool_size, opts->bdev_io_cache_size,
|
|
spdk_thread_get_count());
|
|
SPDK_ERRLOG("bdev_io_pool_size must be at least %" PRIu32 "\n", min_pool_size);
|
|
return -1;
|
|
}
|
|
|
|
g_bdev_opts = *opts;
|
|
return 0;
|
|
}
|
|
|
|
struct spdk_bdev *
|
|
spdk_bdev_first(void)
|
|
{
|
|
struct spdk_bdev *bdev;
|
|
|
|
bdev = TAILQ_FIRST(&g_bdev_mgr.bdevs);
|
|
if (bdev) {
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Starting bdev iteration at %s\n", bdev->name);
|
|
}
|
|
|
|
return bdev;
|
|
}
|
|
|
|
struct spdk_bdev *
|
|
spdk_bdev_next(struct spdk_bdev *prev)
|
|
{
|
|
struct spdk_bdev *bdev;
|
|
|
|
bdev = TAILQ_NEXT(prev, internal.link);
|
|
if (bdev) {
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Continuing bdev iteration at %s\n", bdev->name);
|
|
}
|
|
|
|
return bdev;
|
|
}
|
|
|
|
static struct spdk_bdev *
|
|
_bdev_next_leaf(struct spdk_bdev *bdev)
|
|
{
|
|
while (bdev != NULL) {
|
|
if (bdev->internal.claim_module == NULL) {
|
|
return bdev;
|
|
} else {
|
|
bdev = TAILQ_NEXT(bdev, internal.link);
|
|
}
|
|
}
|
|
|
|
return bdev;
|
|
}
|
|
|
|
struct spdk_bdev *
|
|
spdk_bdev_first_leaf(void)
|
|
{
|
|
struct spdk_bdev *bdev;
|
|
|
|
bdev = _bdev_next_leaf(TAILQ_FIRST(&g_bdev_mgr.bdevs));
|
|
|
|
if (bdev) {
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Starting bdev iteration at %s\n", bdev->name);
|
|
}
|
|
|
|
return bdev;
|
|
}
|
|
|
|
struct spdk_bdev *
|
|
spdk_bdev_next_leaf(struct spdk_bdev *prev)
|
|
{
|
|
struct spdk_bdev *bdev;
|
|
|
|
bdev = _bdev_next_leaf(TAILQ_NEXT(prev, internal.link));
|
|
|
|
if (bdev) {
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Continuing bdev iteration at %s\n", bdev->name);
|
|
}
|
|
|
|
return bdev;
|
|
}
|
|
|
|
struct spdk_bdev *
|
|
spdk_bdev_get_by_name(const char *bdev_name)
|
|
{
|
|
struct spdk_bdev_alias *tmp;
|
|
struct spdk_bdev *bdev = spdk_bdev_first();
|
|
|
|
while (bdev != NULL) {
|
|
if (strcmp(bdev_name, bdev->name) == 0) {
|
|
return bdev;
|
|
}
|
|
|
|
TAILQ_FOREACH(tmp, &bdev->aliases, tailq) {
|
|
if (strcmp(bdev_name, tmp->alias) == 0) {
|
|
return bdev;
|
|
}
|
|
}
|
|
|
|
bdev = spdk_bdev_next(bdev);
|
|
}
|
|
|
|
return NULL;
|
|
}
|
|
|
|
void
|
|
spdk_bdev_io_set_buf(struct spdk_bdev_io *bdev_io, void *buf, size_t len)
|
|
{
|
|
struct iovec *iovs;
|
|
|
|
iovs = bdev_io->u.bdev.iovs;
|
|
|
|
assert(iovs != NULL);
|
|
assert(bdev_io->u.bdev.iovcnt >= 1);
|
|
|
|
iovs[0].iov_base = buf;
|
|
iovs[0].iov_len = len;
|
|
}
|
|
|
|
static bool
|
|
_is_buf_allocated(struct iovec *iovs)
|
|
{
|
|
return iovs[0].iov_base != NULL;
|
|
}
|
|
|
|
static bool
|
|
_are_iovs_aligned(struct iovec *iovs, int iovcnt, uint32_t alignment)
|
|
{
|
|
int i;
|
|
uintptr_t iov_base;
|
|
|
|
if (spdk_likely(alignment == 1)) {
|
|
return true;
|
|
}
|
|
|
|
for (i = 0; i < iovcnt; i++) {
|
|
iov_base = (uintptr_t)iovs[i].iov_base;
|
|
if ((iov_base & (alignment - 1)) != 0) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
static void
|
|
_copy_iovs_to_buf(void *buf, size_t buf_len, struct iovec *iovs, int iovcnt)
|
|
{
|
|
int i;
|
|
size_t len;
|
|
|
|
for (i = 0; i < iovcnt; i++) {
|
|
len = spdk_min(iovs[i].iov_len, buf_len);
|
|
memcpy(buf, iovs[i].iov_base, len);
|
|
buf += len;
|
|
buf_len -= len;
|
|
}
|
|
}
|
|
|
|
static void
|
|
_copy_buf_to_iovs(struct iovec *iovs, int iovcnt, void *buf, size_t buf_len)
|
|
{
|
|
int i;
|
|
size_t len;
|
|
|
|
for (i = 0; i < iovcnt; i++) {
|
|
len = spdk_min(iovs[i].iov_len, buf_len);
|
|
memcpy(iovs[i].iov_base, buf, len);
|
|
buf += len;
|
|
buf_len -= len;
|
|
}
|
|
}
|
|
|
|
static void
|
|
_bdev_io_set_bounce_buf(struct spdk_bdev_io *bdev_io, void *buf, size_t len)
|
|
{
|
|
/* save original iovec */
|
|
bdev_io->internal.orig_iovs = bdev_io->u.bdev.iovs;
|
|
bdev_io->internal.orig_iovcnt = bdev_io->u.bdev.iovcnt;
|
|
/* set bounce iov */
|
|
bdev_io->u.bdev.iovs = &bdev_io->internal.bounce_iov;
|
|
bdev_io->u.bdev.iovcnt = 1;
|
|
/* set bounce buffer for this operation */
|
|
bdev_io->u.bdev.iovs[0].iov_base = buf;
|
|
bdev_io->u.bdev.iovs[0].iov_len = len;
|
|
/* if this is write path, copy data from original buffer to bounce buffer */
|
|
if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) {
|
|
_copy_iovs_to_buf(buf, len, bdev_io->internal.orig_iovs, bdev_io->internal.orig_iovcnt);
|
|
}
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_io_put_buf(struct spdk_bdev_io *bdev_io)
|
|
{
|
|
struct spdk_mempool *pool;
|
|
struct spdk_bdev_io *tmp;
|
|
void *buf, *aligned_buf;
|
|
bdev_io_stailq_t *stailq;
|
|
struct spdk_bdev_mgmt_channel *ch;
|
|
uint64_t buf_len;
|
|
uint64_t alignment;
|
|
bool buf_allocated;
|
|
|
|
buf = bdev_io->internal.buf;
|
|
buf_len = bdev_io->internal.buf_len;
|
|
alignment = spdk_bdev_get_buf_align(bdev_io->bdev);
|
|
ch = bdev_io->internal.ch->shared_resource->mgmt_ch;
|
|
|
|
bdev_io->internal.buf = NULL;
|
|
|
|
if (buf_len + alignment <= SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_SMALL_BUF_MAX_SIZE) +
|
|
SPDK_BDEV_POOL_ALIGNMENT) {
|
|
pool = g_bdev_mgr.buf_small_pool;
|
|
stailq = &ch->need_buf_small;
|
|
} else {
|
|
pool = g_bdev_mgr.buf_large_pool;
|
|
stailq = &ch->need_buf_large;
|
|
}
|
|
|
|
if (STAILQ_EMPTY(stailq)) {
|
|
spdk_mempool_put(pool, buf);
|
|
} else {
|
|
tmp = STAILQ_FIRST(stailq);
|
|
|
|
alignment = spdk_bdev_get_buf_align(tmp->bdev);
|
|
buf_allocated = _is_buf_allocated(tmp->u.bdev.iovs);
|
|
|
|
aligned_buf = (void *)(((uintptr_t)buf +
|
|
(alignment - 1)) & ~(alignment - 1));
|
|
if (buf_allocated) {
|
|
_bdev_io_set_bounce_buf(tmp, aligned_buf, tmp->internal.buf_len);
|
|
} else {
|
|
spdk_bdev_io_set_buf(tmp, aligned_buf, tmp->internal.buf_len);
|
|
}
|
|
|
|
STAILQ_REMOVE_HEAD(stailq, internal.buf_link);
|
|
tmp->internal.buf = buf;
|
|
tmp->internal.get_buf_cb(tmp->internal.ch->channel, tmp, true);
|
|
}
|
|
}
|
|
|
|
static void
|
|
_bdev_io_unset_bounce_buf(struct spdk_bdev_io *bdev_io)
|
|
{
|
|
/* if this is read path, copy data from bounce buffer to original buffer */
|
|
if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ &&
|
|
bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS) {
|
|
_copy_buf_to_iovs(bdev_io->internal.orig_iovs, bdev_io->internal.orig_iovcnt,
|
|
bdev_io->internal.bounce_iov.iov_base, bdev_io->internal.bounce_iov.iov_len);
|
|
}
|
|
/* set orignal buffer for this io */
|
|
bdev_io->u.bdev.iovcnt = bdev_io->internal.orig_iovcnt;
|
|
bdev_io->u.bdev.iovs = bdev_io->internal.orig_iovs;
|
|
/* disable bouncing buffer for this io */
|
|
bdev_io->internal.orig_iovcnt = 0;
|
|
bdev_io->internal.orig_iovs = NULL;
|
|
/* return bounce buffer to the pool */
|
|
spdk_bdev_io_put_buf(bdev_io);
|
|
}
|
|
|
|
void
|
|
spdk_bdev_io_get_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_buf_cb cb, uint64_t len)
|
|
{
|
|
struct spdk_mempool *pool;
|
|
bdev_io_stailq_t *stailq;
|
|
void *buf, *aligned_buf;
|
|
struct spdk_bdev_mgmt_channel *mgmt_ch;
|
|
uint64_t alignment;
|
|
bool buf_allocated;
|
|
|
|
assert(cb != NULL);
|
|
assert(bdev_io->u.bdev.iovs != NULL);
|
|
|
|
alignment = spdk_bdev_get_buf_align(bdev_io->bdev);
|
|
buf_allocated = _is_buf_allocated(bdev_io->u.bdev.iovs);
|
|
|
|
if (buf_allocated &&
|
|
_are_iovs_aligned(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, alignment)) {
|
|
/* Buffer already present and aligned */
|
|
cb(bdev_io->internal.ch->channel, bdev_io, true);
|
|
return;
|
|
}
|
|
|
|
if (len + alignment > SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_LARGE_BUF_MAX_SIZE) +
|
|
SPDK_BDEV_POOL_ALIGNMENT) {
|
|
SPDK_ERRLOG("Length + alignment %" PRIu64 " is larger than allowed\n",
|
|
len + alignment);
|
|
cb(bdev_io->internal.ch->channel, bdev_io, false);
|
|
return;
|
|
}
|
|
|
|
mgmt_ch = bdev_io->internal.ch->shared_resource->mgmt_ch;
|
|
|
|
bdev_io->internal.buf_len = len;
|
|
bdev_io->internal.get_buf_cb = cb;
|
|
|
|
if (len + alignment <= SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_SMALL_BUF_MAX_SIZE) +
|
|
SPDK_BDEV_POOL_ALIGNMENT) {
|
|
pool = g_bdev_mgr.buf_small_pool;
|
|
stailq = &mgmt_ch->need_buf_small;
|
|
} else {
|
|
pool = g_bdev_mgr.buf_large_pool;
|
|
stailq = &mgmt_ch->need_buf_large;
|
|
}
|
|
|
|
buf = spdk_mempool_get(pool);
|
|
|
|
if (!buf) {
|
|
STAILQ_INSERT_TAIL(stailq, bdev_io, internal.buf_link);
|
|
} else {
|
|
aligned_buf = (void *)(((uintptr_t)buf + (alignment - 1)) & ~(alignment - 1));
|
|
|
|
if (buf_allocated) {
|
|
_bdev_io_set_bounce_buf(bdev_io, aligned_buf, len);
|
|
} else {
|
|
spdk_bdev_io_set_buf(bdev_io, aligned_buf, len);
|
|
}
|
|
bdev_io->internal.buf = buf;
|
|
bdev_io->internal.get_buf_cb(bdev_io->internal.ch->channel, bdev_io, true);
|
|
}
|
|
}
|
|
|
|
static int
|
|
spdk_bdev_module_get_max_ctx_size(void)
|
|
{
|
|
struct spdk_bdev_module *bdev_module;
|
|
int max_bdev_module_size = 0;
|
|
|
|
TAILQ_FOREACH(bdev_module, &g_bdev_mgr.bdev_modules, internal.tailq) {
|
|
if (bdev_module->get_ctx_size && bdev_module->get_ctx_size() > max_bdev_module_size) {
|
|
max_bdev_module_size = bdev_module->get_ctx_size();
|
|
}
|
|
}
|
|
|
|
return max_bdev_module_size;
|
|
}
|
|
|
|
void
|
|
spdk_bdev_config_text(FILE *fp)
|
|
{
|
|
struct spdk_bdev_module *bdev_module;
|
|
|
|
TAILQ_FOREACH(bdev_module, &g_bdev_mgr.bdev_modules, internal.tailq) {
|
|
if (bdev_module->config_text) {
|
|
bdev_module->config_text(fp);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_qos_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
|
|
{
|
|
int i;
|
|
struct spdk_bdev_qos *qos = bdev->internal.qos;
|
|
uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES];
|
|
|
|
if (!qos) {
|
|
return;
|
|
}
|
|
|
|
spdk_bdev_get_qos_rate_limits(bdev, limits);
|
|
|
|
spdk_json_write_object_begin(w);
|
|
spdk_json_write_named_string(w, "method", "set_bdev_qos_limit");
|
|
|
|
spdk_json_write_named_object_begin(w, "params");
|
|
spdk_json_write_named_string(w, "name", bdev->name);
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
if (limits[i] > 0) {
|
|
spdk_json_write_named_uint64(w, qos_rpc_type[i], limits[i]);
|
|
}
|
|
}
|
|
spdk_json_write_object_end(w);
|
|
|
|
spdk_json_write_object_end(w);
|
|
}
|
|
|
|
void
|
|
spdk_bdev_subsystem_config_json(struct spdk_json_write_ctx *w)
|
|
{
|
|
struct spdk_bdev_module *bdev_module;
|
|
struct spdk_bdev *bdev;
|
|
|
|
assert(w != NULL);
|
|
|
|
spdk_json_write_array_begin(w);
|
|
|
|
spdk_json_write_object_begin(w);
|
|
spdk_json_write_named_string(w, "method", "set_bdev_options");
|
|
spdk_json_write_named_object_begin(w, "params");
|
|
spdk_json_write_named_uint32(w, "bdev_io_pool_size", g_bdev_opts.bdev_io_pool_size);
|
|
spdk_json_write_named_uint32(w, "bdev_io_cache_size", g_bdev_opts.bdev_io_cache_size);
|
|
spdk_json_write_object_end(w);
|
|
spdk_json_write_object_end(w);
|
|
|
|
TAILQ_FOREACH(bdev_module, &g_bdev_mgr.bdev_modules, internal.tailq) {
|
|
if (bdev_module->config_json) {
|
|
bdev_module->config_json(w);
|
|
}
|
|
}
|
|
|
|
TAILQ_FOREACH(bdev, &g_bdev_mgr.bdevs, internal.link) {
|
|
spdk_bdev_qos_config_json(bdev, w);
|
|
|
|
if (bdev->fn_table->write_config_json) {
|
|
bdev->fn_table->write_config_json(bdev, w);
|
|
}
|
|
}
|
|
|
|
spdk_json_write_array_end(w);
|
|
}
|
|
|
|
static int
|
|
spdk_bdev_mgmt_channel_create(void *io_device, void *ctx_buf)
|
|
{
|
|
struct spdk_bdev_mgmt_channel *ch = ctx_buf;
|
|
struct spdk_bdev_io *bdev_io;
|
|
uint32_t i;
|
|
|
|
STAILQ_INIT(&ch->need_buf_small);
|
|
STAILQ_INIT(&ch->need_buf_large);
|
|
|
|
STAILQ_INIT(&ch->per_thread_cache);
|
|
ch->bdev_io_cache_size = g_bdev_opts.bdev_io_cache_size;
|
|
|
|
/* Pre-populate bdev_io cache to ensure this thread cannot be starved. */
|
|
ch->per_thread_cache_count = 0;
|
|
for (i = 0; i < ch->bdev_io_cache_size; i++) {
|
|
bdev_io = spdk_mempool_get(g_bdev_mgr.bdev_io_pool);
|
|
assert(bdev_io != NULL);
|
|
ch->per_thread_cache_count++;
|
|
STAILQ_INSERT_HEAD(&ch->per_thread_cache, bdev_io, internal.buf_link);
|
|
}
|
|
|
|
TAILQ_INIT(&ch->shared_resources);
|
|
TAILQ_INIT(&ch->io_wait_queue);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_mgmt_channel_destroy(void *io_device, void *ctx_buf)
|
|
{
|
|
struct spdk_bdev_mgmt_channel *ch = ctx_buf;
|
|
struct spdk_bdev_io *bdev_io;
|
|
|
|
if (!STAILQ_EMPTY(&ch->need_buf_small) || !STAILQ_EMPTY(&ch->need_buf_large)) {
|
|
SPDK_ERRLOG("Pending I/O list wasn't empty on mgmt channel free\n");
|
|
}
|
|
|
|
if (!TAILQ_EMPTY(&ch->shared_resources)) {
|
|
SPDK_ERRLOG("Module channel list wasn't empty on mgmt channel free\n");
|
|
}
|
|
|
|
while (!STAILQ_EMPTY(&ch->per_thread_cache)) {
|
|
bdev_io = STAILQ_FIRST(&ch->per_thread_cache);
|
|
STAILQ_REMOVE_HEAD(&ch->per_thread_cache, internal.buf_link);
|
|
ch->per_thread_cache_count--;
|
|
spdk_mempool_put(g_bdev_mgr.bdev_io_pool, (void *)bdev_io);
|
|
}
|
|
|
|
assert(ch->per_thread_cache_count == 0);
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_init_complete(int rc)
|
|
{
|
|
spdk_bdev_init_cb cb_fn = g_init_cb_fn;
|
|
void *cb_arg = g_init_cb_arg;
|
|
struct spdk_bdev_module *m;
|
|
|
|
g_bdev_mgr.init_complete = true;
|
|
g_init_cb_fn = NULL;
|
|
g_init_cb_arg = NULL;
|
|
|
|
/*
|
|
* For modules that need to know when subsystem init is complete,
|
|
* inform them now.
|
|
*/
|
|
if (rc == 0) {
|
|
TAILQ_FOREACH(m, &g_bdev_mgr.bdev_modules, internal.tailq) {
|
|
if (m->init_complete) {
|
|
m->init_complete();
|
|
}
|
|
}
|
|
}
|
|
|
|
cb_fn(cb_arg, rc);
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_module_action_complete(void)
|
|
{
|
|
struct spdk_bdev_module *m;
|
|
|
|
/*
|
|
* Don't finish bdev subsystem initialization if
|
|
* module pre-initialization is still in progress, or
|
|
* the subsystem been already initialized.
|
|
*/
|
|
if (!g_bdev_mgr.module_init_complete || g_bdev_mgr.init_complete) {
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* Check all bdev modules for inits/examinations in progress. If any
|
|
* exist, return immediately since we cannot finish bdev subsystem
|
|
* initialization until all are completed.
|
|
*/
|
|
TAILQ_FOREACH(m, &g_bdev_mgr.bdev_modules, internal.tailq) {
|
|
if (m->internal.action_in_progress > 0) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Modules already finished initialization - now that all
|
|
* the bdev modules have finished their asynchronous I/O
|
|
* processing, the entire bdev layer can be marked as complete.
|
|
*/
|
|
spdk_bdev_init_complete(0);
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_module_action_done(struct spdk_bdev_module *module)
|
|
{
|
|
assert(module->internal.action_in_progress > 0);
|
|
module->internal.action_in_progress--;
|
|
spdk_bdev_module_action_complete();
|
|
}
|
|
|
|
void
|
|
spdk_bdev_module_init_done(struct spdk_bdev_module *module)
|
|
{
|
|
spdk_bdev_module_action_done(module);
|
|
}
|
|
|
|
void
|
|
spdk_bdev_module_examine_done(struct spdk_bdev_module *module)
|
|
{
|
|
spdk_bdev_module_action_done(module);
|
|
}
|
|
|
|
/** The last initialized bdev module */
|
|
static struct spdk_bdev_module *g_resume_bdev_module = NULL;
|
|
|
|
static int
|
|
spdk_bdev_modules_init(void)
|
|
{
|
|
struct spdk_bdev_module *module;
|
|
int rc = 0;
|
|
|
|
TAILQ_FOREACH(module, &g_bdev_mgr.bdev_modules, internal.tailq) {
|
|
g_resume_bdev_module = module;
|
|
rc = module->module_init();
|
|
if (rc != 0) {
|
|
return rc;
|
|
}
|
|
}
|
|
|
|
g_resume_bdev_module = NULL;
|
|
return 0;
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_init_failed(void *cb_arg)
|
|
{
|
|
spdk_bdev_init_complete(-1);
|
|
}
|
|
|
|
void
|
|
spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg)
|
|
{
|
|
struct spdk_conf_section *sp;
|
|
struct spdk_bdev_opts bdev_opts;
|
|
int32_t bdev_io_pool_size, bdev_io_cache_size;
|
|
int cache_size;
|
|
int rc = 0;
|
|
char mempool_name[32];
|
|
|
|
assert(cb_fn != NULL);
|
|
|
|
sp = spdk_conf_find_section(NULL, "Bdev");
|
|
if (sp != NULL) {
|
|
spdk_bdev_get_opts(&bdev_opts);
|
|
|
|
bdev_io_pool_size = spdk_conf_section_get_intval(sp, "BdevIoPoolSize");
|
|
if (bdev_io_pool_size >= 0) {
|
|
bdev_opts.bdev_io_pool_size = bdev_io_pool_size;
|
|
}
|
|
|
|
bdev_io_cache_size = spdk_conf_section_get_intval(sp, "BdevIoCacheSize");
|
|
if (bdev_io_cache_size >= 0) {
|
|
bdev_opts.bdev_io_cache_size = bdev_io_cache_size;
|
|
}
|
|
|
|
if (spdk_bdev_set_opts(&bdev_opts)) {
|
|
spdk_bdev_init_complete(-1);
|
|
return;
|
|
}
|
|
|
|
assert(memcmp(&bdev_opts, &g_bdev_opts, sizeof(bdev_opts)) == 0);
|
|
}
|
|
|
|
g_init_cb_fn = cb_fn;
|
|
g_init_cb_arg = cb_arg;
|
|
|
|
snprintf(mempool_name, sizeof(mempool_name), "bdev_io_%d", getpid());
|
|
|
|
g_bdev_mgr.bdev_io_pool = spdk_mempool_create(mempool_name,
|
|
g_bdev_opts.bdev_io_pool_size,
|
|
sizeof(struct spdk_bdev_io) +
|
|
spdk_bdev_module_get_max_ctx_size(),
|
|
0,
|
|
SPDK_ENV_SOCKET_ID_ANY);
|
|
|
|
if (g_bdev_mgr.bdev_io_pool == NULL) {
|
|
SPDK_ERRLOG("could not allocate spdk_bdev_io pool\n");
|
|
spdk_bdev_init_complete(-1);
|
|
return;
|
|
}
|
|
|
|
/**
|
|
* Ensure no more than half of the total buffers end up local caches, by
|
|
* using spdk_thread_get_count() to determine how many local caches we need
|
|
* to account for.
|
|
*/
|
|
cache_size = BUF_SMALL_POOL_SIZE / (2 * spdk_thread_get_count());
|
|
snprintf(mempool_name, sizeof(mempool_name), "buf_small_pool_%d", getpid());
|
|
|
|
g_bdev_mgr.buf_small_pool = spdk_mempool_create(mempool_name,
|
|
BUF_SMALL_POOL_SIZE,
|
|
SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_SMALL_BUF_MAX_SIZE) +
|
|
SPDK_BDEV_POOL_ALIGNMENT,
|
|
cache_size,
|
|
SPDK_ENV_SOCKET_ID_ANY);
|
|
if (!g_bdev_mgr.buf_small_pool) {
|
|
SPDK_ERRLOG("create rbuf small pool failed\n");
|
|
spdk_bdev_init_complete(-1);
|
|
return;
|
|
}
|
|
|
|
cache_size = BUF_LARGE_POOL_SIZE / (2 * spdk_thread_get_count());
|
|
snprintf(mempool_name, sizeof(mempool_name), "buf_large_pool_%d", getpid());
|
|
|
|
g_bdev_mgr.buf_large_pool = spdk_mempool_create(mempool_name,
|
|
BUF_LARGE_POOL_SIZE,
|
|
SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_LARGE_BUF_MAX_SIZE) +
|
|
SPDK_BDEV_POOL_ALIGNMENT,
|
|
cache_size,
|
|
SPDK_ENV_SOCKET_ID_ANY);
|
|
if (!g_bdev_mgr.buf_large_pool) {
|
|
SPDK_ERRLOG("create rbuf large pool failed\n");
|
|
spdk_bdev_init_complete(-1);
|
|
return;
|
|
}
|
|
|
|
g_bdev_mgr.zero_buffer = spdk_dma_zmalloc(ZERO_BUFFER_SIZE, ZERO_BUFFER_SIZE,
|
|
NULL);
|
|
if (!g_bdev_mgr.zero_buffer) {
|
|
SPDK_ERRLOG("create bdev zero buffer failed\n");
|
|
spdk_bdev_init_complete(-1);
|
|
return;
|
|
}
|
|
|
|
#ifdef SPDK_CONFIG_VTUNE
|
|
g_bdev_mgr.domain = __itt_domain_create("spdk_bdev");
|
|
#endif
|
|
|
|
spdk_io_device_register(&g_bdev_mgr, spdk_bdev_mgmt_channel_create,
|
|
spdk_bdev_mgmt_channel_destroy,
|
|
sizeof(struct spdk_bdev_mgmt_channel),
|
|
"bdev_mgr");
|
|
|
|
rc = spdk_bdev_modules_init();
|
|
g_bdev_mgr.module_init_complete = true;
|
|
if (rc != 0) {
|
|
SPDK_ERRLOG("bdev modules init failed\n");
|
|
spdk_thread_send_msg(spdk_get_thread(), spdk_bdev_init_failed, NULL);
|
|
return;
|
|
}
|
|
|
|
spdk_bdev_module_action_complete();
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_mgr_unregister_cb(void *io_device)
|
|
{
|
|
spdk_bdev_fini_cb cb_fn = g_fini_cb_fn;
|
|
|
|
if (spdk_mempool_count(g_bdev_mgr.bdev_io_pool) != g_bdev_opts.bdev_io_pool_size) {
|
|
SPDK_ERRLOG("bdev IO pool count is %zu but should be %u\n",
|
|
spdk_mempool_count(g_bdev_mgr.bdev_io_pool),
|
|
g_bdev_opts.bdev_io_pool_size);
|
|
}
|
|
|
|
if (spdk_mempool_count(g_bdev_mgr.buf_small_pool) != BUF_SMALL_POOL_SIZE) {
|
|
SPDK_ERRLOG("Small buffer pool count is %zu but should be %u\n",
|
|
spdk_mempool_count(g_bdev_mgr.buf_small_pool),
|
|
BUF_SMALL_POOL_SIZE);
|
|
assert(false);
|
|
}
|
|
|
|
if (spdk_mempool_count(g_bdev_mgr.buf_large_pool) != BUF_LARGE_POOL_SIZE) {
|
|
SPDK_ERRLOG("Large buffer pool count is %zu but should be %u\n",
|
|
spdk_mempool_count(g_bdev_mgr.buf_large_pool),
|
|
BUF_LARGE_POOL_SIZE);
|
|
assert(false);
|
|
}
|
|
|
|
spdk_mempool_free(g_bdev_mgr.bdev_io_pool);
|
|
spdk_mempool_free(g_bdev_mgr.buf_small_pool);
|
|
spdk_mempool_free(g_bdev_mgr.buf_large_pool);
|
|
spdk_dma_free(g_bdev_mgr.zero_buffer);
|
|
|
|
cb_fn(g_fini_cb_arg);
|
|
g_fini_cb_fn = NULL;
|
|
g_fini_cb_arg = NULL;
|
|
g_bdev_mgr.init_complete = false;
|
|
g_bdev_mgr.module_init_complete = false;
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_module_finish_iter(void *arg)
|
|
{
|
|
struct spdk_bdev_module *bdev_module;
|
|
|
|
/* Start iterating from the last touched module */
|
|
if (!g_resume_bdev_module) {
|
|
bdev_module = TAILQ_LAST(&g_bdev_mgr.bdev_modules, bdev_module_list);
|
|
} else {
|
|
bdev_module = TAILQ_PREV(g_resume_bdev_module, bdev_module_list,
|
|
internal.tailq);
|
|
}
|
|
|
|
while (bdev_module) {
|
|
if (bdev_module->async_fini) {
|
|
/* Save our place so we can resume later. We must
|
|
* save the variable here, before calling module_fini()
|
|
* below, because in some cases the module may immediately
|
|
* call spdk_bdev_module_finish_done() and re-enter
|
|
* this function to continue iterating. */
|
|
g_resume_bdev_module = bdev_module;
|
|
}
|
|
|
|
if (bdev_module->module_fini) {
|
|
bdev_module->module_fini();
|
|
}
|
|
|
|
if (bdev_module->async_fini) {
|
|
return;
|
|
}
|
|
|
|
bdev_module = TAILQ_PREV(bdev_module, bdev_module_list,
|
|
internal.tailq);
|
|
}
|
|
|
|
g_resume_bdev_module = NULL;
|
|
spdk_io_device_unregister(&g_bdev_mgr, spdk_bdev_mgr_unregister_cb);
|
|
}
|
|
|
|
void
|
|
spdk_bdev_module_finish_done(void)
|
|
{
|
|
if (spdk_get_thread() != g_fini_thread) {
|
|
spdk_thread_send_msg(g_fini_thread, spdk_bdev_module_finish_iter, NULL);
|
|
} else {
|
|
spdk_bdev_module_finish_iter(NULL);
|
|
}
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_finish_unregister_bdevs_iter(void *cb_arg, int bdeverrno)
|
|
{
|
|
struct spdk_bdev *bdev = cb_arg;
|
|
|
|
if (bdeverrno && bdev) {
|
|
SPDK_WARNLOG("Unable to unregister bdev '%s' during spdk_bdev_finish()\n",
|
|
bdev->name);
|
|
|
|
/*
|
|
* Since the call to spdk_bdev_unregister() failed, we have no way to free this
|
|
* bdev; try to continue by manually removing this bdev from the list and continue
|
|
* with the next bdev in the list.
|
|
*/
|
|
TAILQ_REMOVE(&g_bdev_mgr.bdevs, bdev, internal.link);
|
|
}
|
|
|
|
if (TAILQ_EMPTY(&g_bdev_mgr.bdevs)) {
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Done unregistering bdevs\n");
|
|
/*
|
|
* Bdev module finish need to be deferred as we might be in the middle of some context
|
|
* (like bdev part free) that will use this bdev (or private bdev driver ctx data)
|
|
* after returning.
|
|
*/
|
|
spdk_thread_send_msg(spdk_get_thread(), spdk_bdev_module_finish_iter, NULL);
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* Unregister last unclaimed bdev in the list, to ensure that bdev subsystem
|
|
* shutdown proceeds top-down. The goal is to give virtual bdevs an opportunity
|
|
* to detect clean shutdown as opposed to run-time hot removal of the underlying
|
|
* base bdevs.
|
|
*
|
|
* Also, walk the list in the reverse order.
|
|
*/
|
|
for (bdev = TAILQ_LAST(&g_bdev_mgr.bdevs, spdk_bdev_list);
|
|
bdev; bdev = TAILQ_PREV(bdev, spdk_bdev_list, internal.link)) {
|
|
if (bdev->internal.claim_module != NULL) {
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Skipping claimed bdev '%s'(<-'%s').\n",
|
|
bdev->name, bdev->internal.claim_module->name);
|
|
continue;
|
|
}
|
|
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Unregistering bdev '%s'\n", bdev->name);
|
|
spdk_bdev_unregister(bdev, _spdk_bdev_finish_unregister_bdevs_iter, bdev);
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* If any bdev fails to unclaim underlying bdev properly, we may face the
|
|
* case of bdev list consisting of claimed bdevs only (if claims are managed
|
|
* correctly, this would mean there's a loop in the claims graph which is
|
|
* clearly impossible). Warn and unregister last bdev on the list then.
|
|
*/
|
|
for (bdev = TAILQ_LAST(&g_bdev_mgr.bdevs, spdk_bdev_list);
|
|
bdev; bdev = TAILQ_PREV(bdev, spdk_bdev_list, internal.link)) {
|
|
SPDK_ERRLOG("Unregistering claimed bdev '%s'!\n", bdev->name);
|
|
spdk_bdev_unregister(bdev, _spdk_bdev_finish_unregister_bdevs_iter, bdev);
|
|
return;
|
|
}
|
|
}
|
|
|
|
void
|
|
spdk_bdev_finish(spdk_bdev_fini_cb cb_fn, void *cb_arg)
|
|
{
|
|
struct spdk_bdev_module *m;
|
|
|
|
assert(cb_fn != NULL);
|
|
|
|
g_fini_thread = spdk_get_thread();
|
|
|
|
g_fini_cb_fn = cb_fn;
|
|
g_fini_cb_arg = cb_arg;
|
|
|
|
TAILQ_FOREACH(m, &g_bdev_mgr.bdev_modules, internal.tailq) {
|
|
if (m->fini_start) {
|
|
m->fini_start();
|
|
}
|
|
}
|
|
|
|
_spdk_bdev_finish_unregister_bdevs_iter(NULL, 0);
|
|
}
|
|
|
|
static struct spdk_bdev_io *
|
|
spdk_bdev_get_io(struct spdk_bdev_channel *channel)
|
|
{
|
|
struct spdk_bdev_mgmt_channel *ch = channel->shared_resource->mgmt_ch;
|
|
struct spdk_bdev_io *bdev_io;
|
|
|
|
if (ch->per_thread_cache_count > 0) {
|
|
bdev_io = STAILQ_FIRST(&ch->per_thread_cache);
|
|
STAILQ_REMOVE_HEAD(&ch->per_thread_cache, internal.buf_link);
|
|
ch->per_thread_cache_count--;
|
|
} else if (spdk_unlikely(!TAILQ_EMPTY(&ch->io_wait_queue))) {
|
|
/*
|
|
* Don't try to look for bdev_ios in the global pool if there are
|
|
* waiters on bdev_ios - we don't want this caller to jump the line.
|
|
*/
|
|
bdev_io = NULL;
|
|
} else {
|
|
bdev_io = spdk_mempool_get(g_bdev_mgr.bdev_io_pool);
|
|
}
|
|
|
|
return bdev_io;
|
|
}
|
|
|
|
void
|
|
spdk_bdev_free_io(struct spdk_bdev_io *bdev_io)
|
|
{
|
|
struct spdk_bdev_mgmt_channel *ch = bdev_io->internal.ch->shared_resource->mgmt_ch;
|
|
|
|
assert(bdev_io != NULL);
|
|
assert(bdev_io->internal.status != SPDK_BDEV_IO_STATUS_PENDING);
|
|
|
|
if (bdev_io->internal.buf != NULL) {
|
|
spdk_bdev_io_put_buf(bdev_io);
|
|
}
|
|
|
|
if (ch->per_thread_cache_count < ch->bdev_io_cache_size) {
|
|
ch->per_thread_cache_count++;
|
|
STAILQ_INSERT_HEAD(&ch->per_thread_cache, bdev_io, internal.buf_link);
|
|
while (ch->per_thread_cache_count > 0 && !TAILQ_EMPTY(&ch->io_wait_queue)) {
|
|
struct spdk_bdev_io_wait_entry *entry;
|
|
|
|
entry = TAILQ_FIRST(&ch->io_wait_queue);
|
|
TAILQ_REMOVE(&ch->io_wait_queue, entry, link);
|
|
entry->cb_fn(entry->cb_arg);
|
|
}
|
|
} else {
|
|
/* We should never have a full cache with entries on the io wait queue. */
|
|
assert(TAILQ_EMPTY(&ch->io_wait_queue));
|
|
spdk_mempool_put(g_bdev_mgr.bdev_io_pool, (void *)bdev_io);
|
|
}
|
|
}
|
|
|
|
static bool
|
|
_spdk_bdev_qos_is_iops_rate_limit(enum spdk_bdev_qos_rate_limit_type limit)
|
|
{
|
|
assert(limit != SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES);
|
|
|
|
switch (limit) {
|
|
case SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT:
|
|
return true;
|
|
case SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT:
|
|
case SPDK_BDEV_QOS_R_BPS_RATE_LIMIT:
|
|
case SPDK_BDEV_QOS_W_BPS_RATE_LIMIT:
|
|
return false;
|
|
case SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES:
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
static bool
|
|
_spdk_bdev_qos_io_to_limit(struct spdk_bdev_io *bdev_io)
|
|
{
|
|
switch (bdev_io->type) {
|
|
case SPDK_BDEV_IO_TYPE_NVME_IO:
|
|
case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
|
|
case SPDK_BDEV_IO_TYPE_READ:
|
|
case SPDK_BDEV_IO_TYPE_WRITE:
|
|
return true;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
static bool
|
|
_spdk_bdev_is_read_io(struct spdk_bdev_io *bdev_io)
|
|
{
|
|
switch (bdev_io->type) {
|
|
case SPDK_BDEV_IO_TYPE_NVME_IO:
|
|
case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
|
|
/* Bit 1 (0x2) set for read operation */
|
|
if (bdev_io->u.nvme_passthru.cmd.opc & SPDK_NVME_OPC_READ) {
|
|
return true;
|
|
} else {
|
|
return false;
|
|
}
|
|
case SPDK_BDEV_IO_TYPE_READ:
|
|
return true;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
static uint64_t
|
|
_spdk_bdev_get_io_size_in_byte(struct spdk_bdev_io *bdev_io)
|
|
{
|
|
struct spdk_bdev *bdev = bdev_io->bdev;
|
|
|
|
switch (bdev_io->type) {
|
|
case SPDK_BDEV_IO_TYPE_NVME_IO:
|
|
case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
|
|
return bdev_io->u.nvme_passthru.nbytes;
|
|
case SPDK_BDEV_IO_TYPE_READ:
|
|
case SPDK_BDEV_IO_TYPE_WRITE:
|
|
return bdev_io->u.bdev.num_blocks * bdev->blocklen;
|
|
default:
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
static bool
|
|
_spdk_bdev_qos_rw_queue_io(const struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
|
|
{
|
|
if (limit->max_per_timeslice > 0 && limit->remaining_this_timeslice <= 0) {
|
|
return true;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
static bool
|
|
_spdk_bdev_qos_r_queue_io(const struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
|
|
{
|
|
if (_spdk_bdev_is_read_io(io) == false) {
|
|
return false;
|
|
}
|
|
|
|
return _spdk_bdev_qos_rw_queue_io(limit, io);
|
|
}
|
|
|
|
static bool
|
|
_spdk_bdev_qos_w_queue_io(const struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
|
|
{
|
|
if (_spdk_bdev_is_read_io(io) == true) {
|
|
return false;
|
|
}
|
|
|
|
return _spdk_bdev_qos_rw_queue_io(limit, io);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_qos_rw_iops_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
|
|
{
|
|
limit->remaining_this_timeslice--;
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_qos_rw_bps_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
|
|
{
|
|
limit->remaining_this_timeslice -= _spdk_bdev_get_io_size_in_byte(io);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_qos_r_bps_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
|
|
{
|
|
if (_spdk_bdev_is_read_io(io) == false) {
|
|
return;
|
|
}
|
|
|
|
return _spdk_bdev_qos_rw_bps_update_quota(limit, io);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_qos_w_bps_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
|
|
{
|
|
if (_spdk_bdev_is_read_io(io) == true) {
|
|
return;
|
|
}
|
|
|
|
return _spdk_bdev_qos_rw_bps_update_quota(limit, io);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_qos_set_ops(struct spdk_bdev_qos *qos)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
if (qos->rate_limits[i].limit == SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {
|
|
qos->rate_limits[i].queue_io = NULL;
|
|
qos->rate_limits[i].update_quota = NULL;
|
|
continue;
|
|
}
|
|
|
|
switch (i) {
|
|
case SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT:
|
|
qos->rate_limits[i].queue_io = _spdk_bdev_qos_rw_queue_io;
|
|
qos->rate_limits[i].update_quota = _spdk_bdev_qos_rw_iops_update_quota;
|
|
break;
|
|
case SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT:
|
|
qos->rate_limits[i].queue_io = _spdk_bdev_qos_rw_queue_io;
|
|
qos->rate_limits[i].update_quota = _spdk_bdev_qos_rw_bps_update_quota;
|
|
break;
|
|
case SPDK_BDEV_QOS_R_BPS_RATE_LIMIT:
|
|
qos->rate_limits[i].queue_io = _spdk_bdev_qos_r_queue_io;
|
|
qos->rate_limits[i].update_quota = _spdk_bdev_qos_r_bps_update_quota;
|
|
break;
|
|
case SPDK_BDEV_QOS_W_BPS_RATE_LIMIT:
|
|
qos->rate_limits[i].queue_io = _spdk_bdev_qos_w_queue_io;
|
|
qos->rate_limits[i].update_quota = _spdk_bdev_qos_w_bps_update_quota;
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
static int
|
|
_spdk_bdev_qos_io_submit(struct spdk_bdev_channel *ch, struct spdk_bdev_qos *qos)
|
|
{
|
|
struct spdk_bdev_io *bdev_io = NULL, *tmp = NULL;
|
|
struct spdk_bdev *bdev = ch->bdev;
|
|
struct spdk_bdev_shared_resource *shared_resource = ch->shared_resource;
|
|
int i, submitted_ios = 0;
|
|
|
|
TAILQ_FOREACH_SAFE(bdev_io, &qos->queued, internal.link, tmp) {
|
|
if (_spdk_bdev_qos_io_to_limit(bdev_io) == true) {
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
if (!qos->rate_limits[i].queue_io) {
|
|
continue;
|
|
}
|
|
|
|
if (qos->rate_limits[i].queue_io(&qos->rate_limits[i],
|
|
bdev_io) == true) {
|
|
return submitted_ios;
|
|
}
|
|
}
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
if (!qos->rate_limits[i].update_quota) {
|
|
continue;
|
|
}
|
|
|
|
qos->rate_limits[i].update_quota(&qos->rate_limits[i], bdev_io);
|
|
}
|
|
}
|
|
|
|
TAILQ_REMOVE(&qos->queued, bdev_io, internal.link);
|
|
ch->io_outstanding++;
|
|
shared_resource->io_outstanding++;
|
|
bdev_io->internal.in_submit_request = true;
|
|
bdev->fn_table->submit_request(ch->channel, bdev_io);
|
|
bdev_io->internal.in_submit_request = false;
|
|
submitted_ios++;
|
|
}
|
|
|
|
return submitted_ios;
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_queue_io_wait_with_cb(struct spdk_bdev_io *bdev_io, spdk_bdev_io_wait_cb cb_fn)
|
|
{
|
|
int rc;
|
|
|
|
bdev_io->internal.waitq_entry.bdev = bdev_io->bdev;
|
|
bdev_io->internal.waitq_entry.cb_fn = cb_fn;
|
|
bdev_io->internal.waitq_entry.cb_arg = bdev_io;
|
|
rc = spdk_bdev_queue_io_wait(bdev_io->bdev, spdk_io_channel_from_ctx(bdev_io->internal.ch),
|
|
&bdev_io->internal.waitq_entry);
|
|
if (rc != 0) {
|
|
SPDK_ERRLOG("Queue IO failed, rc=%d\n", rc);
|
|
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
|
|
bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
|
|
}
|
|
}
|
|
|
|
static bool
|
|
_spdk_bdev_io_type_can_split(uint8_t type)
|
|
{
|
|
assert(type != SPDK_BDEV_IO_TYPE_INVALID);
|
|
assert(type < SPDK_BDEV_NUM_IO_TYPES);
|
|
|
|
/* Only split READ and WRITE I/O. Theoretically other types of I/O like
|
|
* UNMAP could be split, but these types of I/O are typically much larger
|
|
* in size (sometimes the size of the entire block device), and the bdev
|
|
* module can more efficiently split these types of I/O. Plus those types
|
|
* of I/O do not have a payload, which makes the splitting process simpler.
|
|
*/
|
|
if (type == SPDK_BDEV_IO_TYPE_READ || type == SPDK_BDEV_IO_TYPE_WRITE) {
|
|
return true;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
static bool
|
|
_spdk_bdev_io_should_split(struct spdk_bdev_io *bdev_io)
|
|
{
|
|
uint64_t start_stripe, end_stripe;
|
|
uint32_t io_boundary = bdev_io->bdev->optimal_io_boundary;
|
|
|
|
if (io_boundary == 0) {
|
|
return false;
|
|
}
|
|
|
|
if (!_spdk_bdev_io_type_can_split(bdev_io->type)) {
|
|
return false;
|
|
}
|
|
|
|
start_stripe = bdev_io->u.bdev.offset_blocks;
|
|
end_stripe = start_stripe + bdev_io->u.bdev.num_blocks - 1;
|
|
/* Avoid expensive div operations if possible. These spdk_u32 functions are very cheap. */
|
|
if (spdk_likely(spdk_u32_is_pow2(io_boundary))) {
|
|
start_stripe >>= spdk_u32log2(io_boundary);
|
|
end_stripe >>= spdk_u32log2(io_boundary);
|
|
} else {
|
|
start_stripe /= io_boundary;
|
|
end_stripe /= io_boundary;
|
|
}
|
|
return (start_stripe != end_stripe);
|
|
}
|
|
|
|
static uint32_t
|
|
_to_next_boundary(uint64_t offset, uint32_t boundary)
|
|
{
|
|
return (boundary - (offset % boundary));
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_io_split_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg);
|
|
|
|
static void
|
|
_spdk_bdev_io_split_with_payload(void *_bdev_io)
|
|
{
|
|
struct spdk_bdev_io *bdev_io = _bdev_io;
|
|
uint64_t current_offset, remaining;
|
|
uint32_t blocklen, to_next_boundary, to_next_boundary_bytes;
|
|
struct iovec *parent_iov, *iov;
|
|
uint64_t parent_iov_offset, iov_len;
|
|
uint32_t parent_iovpos, parent_iovcnt, child_iovcnt, iovcnt;
|
|
int rc;
|
|
|
|
remaining = bdev_io->u.bdev.split_remaining_num_blocks;
|
|
current_offset = bdev_io->u.bdev.split_current_offset_blocks;
|
|
blocklen = bdev_io->bdev->blocklen;
|
|
parent_iov_offset = (current_offset - bdev_io->u.bdev.offset_blocks) * blocklen;
|
|
parent_iovcnt = bdev_io->u.bdev.iovcnt;
|
|
|
|
for (parent_iovpos = 0; parent_iovpos < parent_iovcnt; parent_iovpos++) {
|
|
parent_iov = &bdev_io->u.bdev.iovs[parent_iovpos];
|
|
if (parent_iov_offset < parent_iov->iov_len) {
|
|
break;
|
|
}
|
|
parent_iov_offset -= parent_iov->iov_len;
|
|
}
|
|
|
|
child_iovcnt = 0;
|
|
while (remaining > 0 && parent_iovpos < parent_iovcnt && child_iovcnt < BDEV_IO_NUM_CHILD_IOV) {
|
|
to_next_boundary = _to_next_boundary(current_offset, bdev_io->bdev->optimal_io_boundary);
|
|
to_next_boundary = spdk_min(remaining, to_next_boundary);
|
|
to_next_boundary_bytes = to_next_boundary * blocklen;
|
|
iov = &bdev_io->child_iov[child_iovcnt];
|
|
iovcnt = 0;
|
|
while (to_next_boundary_bytes > 0 && parent_iovpos < parent_iovcnt &&
|
|
child_iovcnt < BDEV_IO_NUM_CHILD_IOV) {
|
|
parent_iov = &bdev_io->u.bdev.iovs[parent_iovpos];
|
|
iov_len = spdk_min(to_next_boundary_bytes, parent_iov->iov_len - parent_iov_offset);
|
|
to_next_boundary_bytes -= iov_len;
|
|
|
|
bdev_io->child_iov[child_iovcnt].iov_base = parent_iov->iov_base + parent_iov_offset;
|
|
bdev_io->child_iov[child_iovcnt].iov_len = iov_len;
|
|
|
|
if (iov_len < parent_iov->iov_len - parent_iov_offset) {
|
|
parent_iov_offset += iov_len;
|
|
} else {
|
|
parent_iovpos++;
|
|
parent_iov_offset = 0;
|
|
}
|
|
child_iovcnt++;
|
|
iovcnt++;
|
|
}
|
|
|
|
if (to_next_boundary_bytes > 0) {
|
|
/* We had to stop this child I/O early because we ran out of
|
|
* child_iov space. Make sure the iovs collected are valid and
|
|
* then adjust to_next_boundary before starting the child I/O.
|
|
*/
|
|
if ((to_next_boundary_bytes % blocklen) != 0) {
|
|
SPDK_ERRLOG("Remaining %" PRIu32 " is not multiple of block size %" PRIu32 "\n",
|
|
to_next_boundary_bytes, blocklen);
|
|
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
|
|
if (bdev_io->u.bdev.split_outstanding == 0) {
|
|
bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
|
|
}
|
|
return;
|
|
}
|
|
to_next_boundary -= to_next_boundary_bytes / blocklen;
|
|
}
|
|
|
|
bdev_io->u.bdev.split_outstanding++;
|
|
|
|
if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ) {
|
|
rc = spdk_bdev_readv_blocks(bdev_io->internal.desc,
|
|
spdk_io_channel_from_ctx(bdev_io->internal.ch),
|
|
iov, iovcnt, current_offset, to_next_boundary,
|
|
_spdk_bdev_io_split_done, bdev_io);
|
|
} else {
|
|
rc = spdk_bdev_writev_blocks(bdev_io->internal.desc,
|
|
spdk_io_channel_from_ctx(bdev_io->internal.ch),
|
|
iov, iovcnt, current_offset, to_next_boundary,
|
|
_spdk_bdev_io_split_done, bdev_io);
|
|
}
|
|
|
|
if (rc == 0) {
|
|
current_offset += to_next_boundary;
|
|
remaining -= to_next_boundary;
|
|
bdev_io->u.bdev.split_current_offset_blocks = current_offset;
|
|
bdev_io->u.bdev.split_remaining_num_blocks = remaining;
|
|
} else {
|
|
bdev_io->u.bdev.split_outstanding--;
|
|
if (rc == -ENOMEM) {
|
|
if (bdev_io->u.bdev.split_outstanding == 0) {
|
|
/* No I/O is outstanding. Hence we should wait here. */
|
|
_spdk_bdev_queue_io_wait_with_cb(bdev_io,
|
|
_spdk_bdev_io_split_with_payload);
|
|
}
|
|
} else {
|
|
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
|
|
if (bdev_io->u.bdev.split_outstanding == 0) {
|
|
bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
|
|
}
|
|
}
|
|
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_io_split_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
|
|
{
|
|
struct spdk_bdev_io *parent_io = cb_arg;
|
|
|
|
spdk_bdev_free_io(bdev_io);
|
|
|
|
if (!success) {
|
|
parent_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
|
|
}
|
|
parent_io->u.bdev.split_outstanding--;
|
|
if (parent_io->u.bdev.split_outstanding != 0) {
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* Parent I/O finishes when all blocks are consumed or there is any failure of
|
|
* child I/O and no outstanding child I/O.
|
|
*/
|
|
if (parent_io->u.bdev.split_remaining_num_blocks == 0 ||
|
|
parent_io->internal.status != SPDK_BDEV_IO_STATUS_SUCCESS) {
|
|
parent_io->internal.cb(parent_io, parent_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS,
|
|
parent_io->internal.caller_ctx);
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* Continue with the splitting process. This function will complete the parent I/O if the
|
|
* splitting is done.
|
|
*/
|
|
_spdk_bdev_io_split_with_payload(parent_io);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_io_split(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
|
|
{
|
|
assert(_spdk_bdev_io_type_can_split(bdev_io->type));
|
|
|
|
bdev_io->u.bdev.split_current_offset_blocks = bdev_io->u.bdev.offset_blocks;
|
|
bdev_io->u.bdev.split_remaining_num_blocks = bdev_io->u.bdev.num_blocks;
|
|
bdev_io->u.bdev.split_outstanding = 0;
|
|
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
|
|
|
|
_spdk_bdev_io_split_with_payload(bdev_io);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_io_split_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io,
|
|
bool success)
|
|
{
|
|
if (!success) {
|
|
spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
|
|
return;
|
|
}
|
|
|
|
_spdk_bdev_io_split(ch, bdev_io);
|
|
}
|
|
|
|
/* Explicitly mark this inline, since it's used as a function pointer and otherwise won't
|
|
* be inlined, at least on some compilers.
|
|
*/
|
|
static inline void
|
|
_spdk_bdev_io_submit(void *ctx)
|
|
{
|
|
struct spdk_bdev_io *bdev_io = ctx;
|
|
struct spdk_bdev *bdev = bdev_io->bdev;
|
|
struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch;
|
|
struct spdk_io_channel *ch = bdev_ch->channel;
|
|
struct spdk_bdev_shared_resource *shared_resource = bdev_ch->shared_resource;
|
|
uint64_t tsc;
|
|
|
|
tsc = spdk_get_ticks();
|
|
bdev_io->internal.submit_tsc = tsc;
|
|
spdk_trace_record_tsc(tsc, TRACE_BDEV_IO_START, 0, 0, (uintptr_t)bdev_io, bdev_io->type);
|
|
bdev_ch->io_outstanding++;
|
|
shared_resource->io_outstanding++;
|
|
bdev_io->internal.in_submit_request = true;
|
|
if (spdk_likely(bdev_ch->flags == 0)) {
|
|
if (spdk_likely(TAILQ_EMPTY(&shared_resource->nomem_io))) {
|
|
bdev->fn_table->submit_request(ch, bdev_io);
|
|
} else {
|
|
bdev_ch->io_outstanding--;
|
|
shared_resource->io_outstanding--;
|
|
TAILQ_INSERT_TAIL(&shared_resource->nomem_io, bdev_io, internal.link);
|
|
}
|
|
} else if (bdev_ch->flags & BDEV_CH_RESET_IN_PROGRESS) {
|
|
spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
|
|
} else if (bdev_ch->flags & BDEV_CH_QOS_ENABLED) {
|
|
bdev_ch->io_outstanding--;
|
|
shared_resource->io_outstanding--;
|
|
TAILQ_INSERT_TAIL(&bdev->internal.qos->queued, bdev_io, internal.link);
|
|
_spdk_bdev_qos_io_submit(bdev_ch, bdev->internal.qos);
|
|
} else {
|
|
SPDK_ERRLOG("unknown bdev_ch flag %x found\n", bdev_ch->flags);
|
|
spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
|
|
}
|
|
bdev_io->internal.in_submit_request = false;
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_io_submit(struct spdk_bdev_io *bdev_io)
|
|
{
|
|
struct spdk_bdev *bdev = bdev_io->bdev;
|
|
struct spdk_thread *thread = spdk_io_channel_get_thread(bdev_io->internal.ch->channel);
|
|
|
|
assert(thread != NULL);
|
|
assert(bdev_io->internal.status == SPDK_BDEV_IO_STATUS_PENDING);
|
|
|
|
if (bdev->split_on_optimal_io_boundary && _spdk_bdev_io_should_split(bdev_io)) {
|
|
if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ) {
|
|
spdk_bdev_io_get_buf(bdev_io, _spdk_bdev_io_split_get_buf_cb,
|
|
bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen);
|
|
} else {
|
|
_spdk_bdev_io_split(NULL, bdev_io);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (bdev_io->internal.ch->flags & BDEV_CH_QOS_ENABLED) {
|
|
if ((thread == bdev->internal.qos->thread) || !bdev->internal.qos->thread) {
|
|
_spdk_bdev_io_submit(bdev_io);
|
|
} else {
|
|
bdev_io->internal.io_submit_ch = bdev_io->internal.ch;
|
|
bdev_io->internal.ch = bdev->internal.qos->ch;
|
|
spdk_thread_send_msg(bdev->internal.qos->thread, _spdk_bdev_io_submit, bdev_io);
|
|
}
|
|
} else {
|
|
_spdk_bdev_io_submit(bdev_io);
|
|
}
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_io_submit_reset(struct spdk_bdev_io *bdev_io)
|
|
{
|
|
struct spdk_bdev *bdev = bdev_io->bdev;
|
|
struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch;
|
|
struct spdk_io_channel *ch = bdev_ch->channel;
|
|
|
|
assert(bdev_io->internal.status == SPDK_BDEV_IO_STATUS_PENDING);
|
|
|
|
bdev_io->internal.in_submit_request = true;
|
|
bdev->fn_table->submit_request(ch, bdev_io);
|
|
bdev_io->internal.in_submit_request = false;
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_io_init(struct spdk_bdev_io *bdev_io,
|
|
struct spdk_bdev *bdev, void *cb_arg,
|
|
spdk_bdev_io_completion_cb cb)
|
|
{
|
|
bdev_io->bdev = bdev;
|
|
bdev_io->internal.caller_ctx = cb_arg;
|
|
bdev_io->internal.cb = cb;
|
|
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_PENDING;
|
|
bdev_io->internal.in_submit_request = false;
|
|
bdev_io->internal.buf = NULL;
|
|
bdev_io->internal.io_submit_ch = NULL;
|
|
bdev_io->internal.orig_iovs = NULL;
|
|
bdev_io->internal.orig_iovcnt = 0;
|
|
}
|
|
|
|
static bool
|
|
_spdk_bdev_io_type_supported(struct spdk_bdev *bdev, enum spdk_bdev_io_type io_type)
|
|
{
|
|
return bdev->fn_table->io_type_supported(bdev->ctxt, io_type);
|
|
}
|
|
|
|
bool
|
|
spdk_bdev_io_type_supported(struct spdk_bdev *bdev, enum spdk_bdev_io_type io_type)
|
|
{
|
|
bool supported;
|
|
|
|
supported = _spdk_bdev_io_type_supported(bdev, io_type);
|
|
|
|
if (!supported) {
|
|
switch (io_type) {
|
|
case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
|
|
/* The bdev layer will emulate write zeroes as long as write is supported. */
|
|
supported = _spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_WRITE);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
return supported;
|
|
}
|
|
|
|
int
|
|
spdk_bdev_dump_info_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
|
|
{
|
|
if (bdev->fn_table->dump_info_json) {
|
|
return bdev->fn_table->dump_info_json(bdev->ctxt, w);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_qos_update_max_quota_per_timeslice(struct spdk_bdev_qos *qos)
|
|
{
|
|
uint32_t max_per_timeslice = 0;
|
|
int i;
|
|
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
if (qos->rate_limits[i].limit == SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {
|
|
qos->rate_limits[i].max_per_timeslice = 0;
|
|
continue;
|
|
}
|
|
|
|
max_per_timeslice = qos->rate_limits[i].limit *
|
|
SPDK_BDEV_QOS_TIMESLICE_IN_USEC / SPDK_SEC_TO_USEC;
|
|
|
|
qos->rate_limits[i].max_per_timeslice = spdk_max(max_per_timeslice,
|
|
qos->rate_limits[i].min_per_timeslice);
|
|
|
|
qos->rate_limits[i].remaining_this_timeslice = qos->rate_limits[i].max_per_timeslice;
|
|
}
|
|
|
|
_spdk_bdev_qos_set_ops(qos);
|
|
}
|
|
|
|
static int
|
|
spdk_bdev_channel_poll_qos(void *arg)
|
|
{
|
|
struct spdk_bdev_qos *qos = arg;
|
|
uint64_t now = spdk_get_ticks();
|
|
int i;
|
|
|
|
if (now < (qos->last_timeslice + qos->timeslice_size)) {
|
|
/* We received our callback earlier than expected - return
|
|
* immediately and wait to do accounting until at least one
|
|
* timeslice has actually expired. This should never happen
|
|
* with a well-behaved timer implementation.
|
|
*/
|
|
return 0;
|
|
}
|
|
|
|
/* Reset for next round of rate limiting */
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
/* We may have allowed the IOs or bytes to slightly overrun in the last
|
|
* timeslice. remaining_this_timeslice is signed, so if it's negative
|
|
* here, we'll account for the overrun so that the next timeslice will
|
|
* be appropriately reduced.
|
|
*/
|
|
if (qos->rate_limits[i].remaining_this_timeslice > 0) {
|
|
qos->rate_limits[i].remaining_this_timeslice = 0;
|
|
}
|
|
}
|
|
|
|
while (now >= (qos->last_timeslice + qos->timeslice_size)) {
|
|
qos->last_timeslice += qos->timeslice_size;
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
qos->rate_limits[i].remaining_this_timeslice +=
|
|
qos->rate_limits[i].max_per_timeslice;
|
|
}
|
|
}
|
|
|
|
return _spdk_bdev_qos_io_submit(qos->ch, qos);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_channel_destroy_resource(struct spdk_bdev_channel *ch)
|
|
{
|
|
struct spdk_bdev_shared_resource *shared_resource;
|
|
|
|
spdk_put_io_channel(ch->channel);
|
|
|
|
shared_resource = ch->shared_resource;
|
|
|
|
assert(ch->io_outstanding == 0);
|
|
assert(shared_resource->ref > 0);
|
|
shared_resource->ref--;
|
|
if (shared_resource->ref == 0) {
|
|
assert(shared_resource->io_outstanding == 0);
|
|
TAILQ_REMOVE(&shared_resource->mgmt_ch->shared_resources, shared_resource, link);
|
|
spdk_put_io_channel(spdk_io_channel_from_ctx(shared_resource->mgmt_ch));
|
|
free(shared_resource);
|
|
}
|
|
}
|
|
|
|
/* Caller must hold bdev->internal.mutex. */
|
|
static void
|
|
_spdk_bdev_enable_qos(struct spdk_bdev *bdev, struct spdk_bdev_channel *ch)
|
|
{
|
|
struct spdk_bdev_qos *qos = bdev->internal.qos;
|
|
int i;
|
|
|
|
/* Rate limiting on this bdev enabled */
|
|
if (qos) {
|
|
if (qos->ch == NULL) {
|
|
struct spdk_io_channel *io_ch;
|
|
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Selecting channel %p as QoS channel for bdev %s on thread %p\n", ch,
|
|
bdev->name, spdk_get_thread());
|
|
|
|
/* No qos channel has been selected, so set one up */
|
|
|
|
/* Take another reference to ch */
|
|
io_ch = spdk_get_io_channel(__bdev_to_io_dev(bdev));
|
|
assert(io_ch != NULL);
|
|
qos->ch = ch;
|
|
|
|
qos->thread = spdk_io_channel_get_thread(io_ch);
|
|
|
|
TAILQ_INIT(&qos->queued);
|
|
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
if (_spdk_bdev_qos_is_iops_rate_limit(i) == true) {
|
|
qos->rate_limits[i].min_per_timeslice =
|
|
SPDK_BDEV_QOS_MIN_IO_PER_TIMESLICE;
|
|
} else {
|
|
qos->rate_limits[i].min_per_timeslice =
|
|
SPDK_BDEV_QOS_MIN_BYTE_PER_TIMESLICE;
|
|
}
|
|
|
|
if (qos->rate_limits[i].limit == 0) {
|
|
qos->rate_limits[i].limit = SPDK_BDEV_QOS_LIMIT_NOT_DEFINED;
|
|
}
|
|
}
|
|
spdk_bdev_qos_update_max_quota_per_timeslice(qos);
|
|
qos->timeslice_size =
|
|
SPDK_BDEV_QOS_TIMESLICE_IN_USEC * spdk_get_ticks_hz() / SPDK_SEC_TO_USEC;
|
|
qos->last_timeslice = spdk_get_ticks();
|
|
qos->poller = spdk_poller_register(spdk_bdev_channel_poll_qos,
|
|
qos,
|
|
SPDK_BDEV_QOS_TIMESLICE_IN_USEC);
|
|
}
|
|
|
|
ch->flags |= BDEV_CH_QOS_ENABLED;
|
|
}
|
|
}
|
|
|
|
static int
|
|
spdk_bdev_channel_create(void *io_device, void *ctx_buf)
|
|
{
|
|
struct spdk_bdev *bdev = __bdev_from_io_dev(io_device);
|
|
struct spdk_bdev_channel *ch = ctx_buf;
|
|
struct spdk_io_channel *mgmt_io_ch;
|
|
struct spdk_bdev_mgmt_channel *mgmt_ch;
|
|
struct spdk_bdev_shared_resource *shared_resource;
|
|
|
|
ch->bdev = bdev;
|
|
ch->channel = bdev->fn_table->get_io_channel(bdev->ctxt);
|
|
if (!ch->channel) {
|
|
return -1;
|
|
}
|
|
|
|
assert(ch->histogram == NULL);
|
|
if (bdev->internal.histogram_enabled) {
|
|
ch->histogram = spdk_histogram_data_alloc();
|
|
if (ch->histogram == NULL) {
|
|
SPDK_ERRLOG("Could not allocate histogram\n");
|
|
}
|
|
}
|
|
|
|
mgmt_io_ch = spdk_get_io_channel(&g_bdev_mgr);
|
|
if (!mgmt_io_ch) {
|
|
spdk_put_io_channel(ch->channel);
|
|
return -1;
|
|
}
|
|
|
|
mgmt_ch = spdk_io_channel_get_ctx(mgmt_io_ch);
|
|
TAILQ_FOREACH(shared_resource, &mgmt_ch->shared_resources, link) {
|
|
if (shared_resource->shared_ch == ch->channel) {
|
|
spdk_put_io_channel(mgmt_io_ch);
|
|
shared_resource->ref++;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (shared_resource == NULL) {
|
|
shared_resource = calloc(1, sizeof(*shared_resource));
|
|
if (shared_resource == NULL) {
|
|
spdk_put_io_channel(ch->channel);
|
|
spdk_put_io_channel(mgmt_io_ch);
|
|
return -1;
|
|
}
|
|
|
|
shared_resource->mgmt_ch = mgmt_ch;
|
|
shared_resource->io_outstanding = 0;
|
|
TAILQ_INIT(&shared_resource->nomem_io);
|
|
shared_resource->nomem_threshold = 0;
|
|
shared_resource->shared_ch = ch->channel;
|
|
shared_resource->ref = 1;
|
|
TAILQ_INSERT_TAIL(&mgmt_ch->shared_resources, shared_resource, link);
|
|
}
|
|
|
|
memset(&ch->stat, 0, sizeof(ch->stat));
|
|
ch->stat.ticks_rate = spdk_get_ticks_hz();
|
|
ch->io_outstanding = 0;
|
|
TAILQ_INIT(&ch->queued_resets);
|
|
ch->flags = 0;
|
|
ch->shared_resource = shared_resource;
|
|
|
|
#ifdef SPDK_CONFIG_VTUNE
|
|
{
|
|
char *name;
|
|
__itt_init_ittlib(NULL, 0);
|
|
name = spdk_sprintf_alloc("spdk_bdev_%s_%p", ch->bdev->name, ch);
|
|
if (!name) {
|
|
_spdk_bdev_channel_destroy_resource(ch);
|
|
return -1;
|
|
}
|
|
ch->handle = __itt_string_handle_create(name);
|
|
free(name);
|
|
ch->start_tsc = spdk_get_ticks();
|
|
ch->interval_tsc = spdk_get_ticks_hz() / 100;
|
|
memset(&ch->prev_stat, 0, sizeof(ch->prev_stat));
|
|
}
|
|
#endif
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
_spdk_bdev_enable_qos(bdev, ch);
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Abort I/O that are waiting on a data buffer. These types of I/O are
|
|
* linked using the spdk_bdev_io internal.buf_link TAILQ_ENTRY.
|
|
*/
|
|
static void
|
|
_spdk_bdev_abort_buf_io(bdev_io_stailq_t *queue, struct spdk_bdev_channel *ch)
|
|
{
|
|
bdev_io_stailq_t tmp;
|
|
struct spdk_bdev_io *bdev_io;
|
|
|
|
STAILQ_INIT(&tmp);
|
|
|
|
while (!STAILQ_EMPTY(queue)) {
|
|
bdev_io = STAILQ_FIRST(queue);
|
|
STAILQ_REMOVE_HEAD(queue, internal.buf_link);
|
|
if (bdev_io->internal.ch == ch) {
|
|
spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
|
|
} else {
|
|
STAILQ_INSERT_TAIL(&tmp, bdev_io, internal.buf_link);
|
|
}
|
|
}
|
|
|
|
STAILQ_SWAP(&tmp, queue, spdk_bdev_io);
|
|
}
|
|
|
|
/*
|
|
* Abort I/O that are queued waiting for submission. These types of I/O are
|
|
* linked using the spdk_bdev_io link TAILQ_ENTRY.
|
|
*/
|
|
static void
|
|
_spdk_bdev_abort_queued_io(bdev_io_tailq_t *queue, struct spdk_bdev_channel *ch)
|
|
{
|
|
struct spdk_bdev_io *bdev_io, *tmp;
|
|
|
|
TAILQ_FOREACH_SAFE(bdev_io, queue, internal.link, tmp) {
|
|
if (bdev_io->internal.ch == ch) {
|
|
TAILQ_REMOVE(queue, bdev_io, internal.link);
|
|
/*
|
|
* spdk_bdev_io_complete() assumes that the completed I/O had
|
|
* been submitted to the bdev module. Since in this case it
|
|
* hadn't, bump io_outstanding to account for the decrement
|
|
* that spdk_bdev_io_complete() will do.
|
|
*/
|
|
if (bdev_io->type != SPDK_BDEV_IO_TYPE_RESET) {
|
|
ch->io_outstanding++;
|
|
ch->shared_resource->io_outstanding++;
|
|
}
|
|
spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_qos_channel_destroy(void *cb_arg)
|
|
{
|
|
struct spdk_bdev_qos *qos = cb_arg;
|
|
|
|
spdk_put_io_channel(spdk_io_channel_from_ctx(qos->ch));
|
|
spdk_poller_unregister(&qos->poller);
|
|
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Free QoS %p.\n", qos);
|
|
|
|
free(qos);
|
|
}
|
|
|
|
static int
|
|
spdk_bdev_qos_destroy(struct spdk_bdev *bdev)
|
|
{
|
|
int i;
|
|
|
|
/*
|
|
* Cleanly shutting down the QoS poller is tricky, because
|
|
* during the asynchronous operation the user could open
|
|
* a new descriptor and create a new channel, spawning
|
|
* a new QoS poller.
|
|
*
|
|
* The strategy is to create a new QoS structure here and swap it
|
|
* in. The shutdown path then continues to refer to the old one
|
|
* until it completes and then releases it.
|
|
*/
|
|
struct spdk_bdev_qos *new_qos, *old_qos;
|
|
|
|
old_qos = bdev->internal.qos;
|
|
|
|
new_qos = calloc(1, sizeof(*new_qos));
|
|
if (!new_qos) {
|
|
SPDK_ERRLOG("Unable to allocate memory to shut down QoS.\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
/* Copy the old QoS data into the newly allocated structure */
|
|
memcpy(new_qos, old_qos, sizeof(*new_qos));
|
|
|
|
/* Zero out the key parts of the QoS structure */
|
|
new_qos->ch = NULL;
|
|
new_qos->thread = NULL;
|
|
new_qos->poller = NULL;
|
|
TAILQ_INIT(&new_qos->queued);
|
|
/*
|
|
* The limit member of spdk_bdev_qos_limit structure is not zeroed.
|
|
* It will be used later for the new QoS structure.
|
|
*/
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
new_qos->rate_limits[i].remaining_this_timeslice = 0;
|
|
new_qos->rate_limits[i].min_per_timeslice = 0;
|
|
new_qos->rate_limits[i].max_per_timeslice = 0;
|
|
}
|
|
|
|
bdev->internal.qos = new_qos;
|
|
|
|
if (old_qos->thread == NULL) {
|
|
free(old_qos);
|
|
} else {
|
|
spdk_thread_send_msg(old_qos->thread, spdk_bdev_qos_channel_destroy,
|
|
old_qos);
|
|
}
|
|
|
|
/* It is safe to continue with destroying the bdev even though the QoS channel hasn't
|
|
* been destroyed yet. The destruction path will end up waiting for the final
|
|
* channel to be put before it releases resources. */
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_io_stat_add(struct spdk_bdev_io_stat *total, struct spdk_bdev_io_stat *add)
|
|
{
|
|
total->bytes_read += add->bytes_read;
|
|
total->num_read_ops += add->num_read_ops;
|
|
total->bytes_written += add->bytes_written;
|
|
total->num_write_ops += add->num_write_ops;
|
|
total->bytes_unmapped += add->bytes_unmapped;
|
|
total->num_unmap_ops += add->num_unmap_ops;
|
|
total->read_latency_ticks += add->read_latency_ticks;
|
|
total->write_latency_ticks += add->write_latency_ticks;
|
|
total->unmap_latency_ticks += add->unmap_latency_ticks;
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_channel_destroy(void *io_device, void *ctx_buf)
|
|
{
|
|
struct spdk_bdev_channel *ch = ctx_buf;
|
|
struct spdk_bdev_mgmt_channel *mgmt_ch;
|
|
struct spdk_bdev_shared_resource *shared_resource = ch->shared_resource;
|
|
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Destroying channel %p for bdev %s on thread %p\n", ch, ch->bdev->name,
|
|
spdk_get_thread());
|
|
|
|
/* This channel is going away, so add its statistics into the bdev so that they don't get lost. */
|
|
pthread_mutex_lock(&ch->bdev->internal.mutex);
|
|
_spdk_bdev_io_stat_add(&ch->bdev->internal.stat, &ch->stat);
|
|
pthread_mutex_unlock(&ch->bdev->internal.mutex);
|
|
|
|
mgmt_ch = shared_resource->mgmt_ch;
|
|
|
|
_spdk_bdev_abort_queued_io(&ch->queued_resets, ch);
|
|
_spdk_bdev_abort_queued_io(&shared_resource->nomem_io, ch);
|
|
_spdk_bdev_abort_buf_io(&mgmt_ch->need_buf_small, ch);
|
|
_spdk_bdev_abort_buf_io(&mgmt_ch->need_buf_large, ch);
|
|
|
|
if (ch->histogram) {
|
|
spdk_histogram_data_free(ch->histogram);
|
|
}
|
|
|
|
_spdk_bdev_channel_destroy_resource(ch);
|
|
}
|
|
|
|
int
|
|
spdk_bdev_alias_add(struct spdk_bdev *bdev, const char *alias)
|
|
{
|
|
struct spdk_bdev_alias *tmp;
|
|
|
|
if (alias == NULL) {
|
|
SPDK_ERRLOG("Empty alias passed\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (spdk_bdev_get_by_name(alias)) {
|
|
SPDK_ERRLOG("Bdev name/alias: %s already exists\n", alias);
|
|
return -EEXIST;
|
|
}
|
|
|
|
tmp = calloc(1, sizeof(*tmp));
|
|
if (tmp == NULL) {
|
|
SPDK_ERRLOG("Unable to allocate alias\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
tmp->alias = strdup(alias);
|
|
if (tmp->alias == NULL) {
|
|
free(tmp);
|
|
SPDK_ERRLOG("Unable to allocate alias\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
TAILQ_INSERT_TAIL(&bdev->aliases, tmp, tailq);
|
|
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
spdk_bdev_alias_del(struct spdk_bdev *bdev, const char *alias)
|
|
{
|
|
struct spdk_bdev_alias *tmp;
|
|
|
|
TAILQ_FOREACH(tmp, &bdev->aliases, tailq) {
|
|
if (strcmp(alias, tmp->alias) == 0) {
|
|
TAILQ_REMOVE(&bdev->aliases, tmp, tailq);
|
|
free(tmp->alias);
|
|
free(tmp);
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
SPDK_INFOLOG(SPDK_LOG_BDEV, "Alias %s does not exists\n", alias);
|
|
|
|
return -ENOENT;
|
|
}
|
|
|
|
void
|
|
spdk_bdev_alias_del_all(struct spdk_bdev *bdev)
|
|
{
|
|
struct spdk_bdev_alias *p, *tmp;
|
|
|
|
TAILQ_FOREACH_SAFE(p, &bdev->aliases, tailq, tmp) {
|
|
TAILQ_REMOVE(&bdev->aliases, p, tailq);
|
|
free(p->alias);
|
|
free(p);
|
|
}
|
|
}
|
|
|
|
struct spdk_io_channel *
|
|
spdk_bdev_get_io_channel(struct spdk_bdev_desc *desc)
|
|
{
|
|
return spdk_get_io_channel(__bdev_to_io_dev(desc->bdev));
|
|
}
|
|
|
|
const char *
|
|
spdk_bdev_get_name(const struct spdk_bdev *bdev)
|
|
{
|
|
return bdev->name;
|
|
}
|
|
|
|
const char *
|
|
spdk_bdev_get_product_name(const struct spdk_bdev *bdev)
|
|
{
|
|
return bdev->product_name;
|
|
}
|
|
|
|
const struct spdk_bdev_aliases_list *
|
|
spdk_bdev_get_aliases(const struct spdk_bdev *bdev)
|
|
{
|
|
return &bdev->aliases;
|
|
}
|
|
|
|
uint32_t
|
|
spdk_bdev_get_block_size(const struct spdk_bdev *bdev)
|
|
{
|
|
return bdev->blocklen;
|
|
}
|
|
|
|
uint64_t
|
|
spdk_bdev_get_num_blocks(const struct spdk_bdev *bdev)
|
|
{
|
|
return bdev->blockcnt;
|
|
}
|
|
|
|
const char *
|
|
spdk_bdev_get_qos_rpc_type(enum spdk_bdev_qos_rate_limit_type type)
|
|
{
|
|
return qos_rpc_type[type];
|
|
}
|
|
|
|
void
|
|
spdk_bdev_get_qos_rate_limits(struct spdk_bdev *bdev, uint64_t *limits)
|
|
{
|
|
int i;
|
|
|
|
memset(limits, 0, sizeof(*limits) * SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES);
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
if (bdev->internal.qos) {
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
if (bdev->internal.qos->rate_limits[i].limit !=
|
|
SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {
|
|
limits[i] = bdev->internal.qos->rate_limits[i].limit;
|
|
if (_spdk_bdev_qos_is_iops_rate_limit(i) == false) {
|
|
/* Change from Byte to Megabyte which is user visible. */
|
|
limits[i] = limits[i] / 1024 / 1024;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
}
|
|
|
|
size_t
|
|
spdk_bdev_get_buf_align(const struct spdk_bdev *bdev)
|
|
{
|
|
return 1 << bdev->required_alignment;
|
|
}
|
|
|
|
uint32_t
|
|
spdk_bdev_get_optimal_io_boundary(const struct spdk_bdev *bdev)
|
|
{
|
|
return bdev->optimal_io_boundary;
|
|
}
|
|
|
|
bool
|
|
spdk_bdev_has_write_cache(const struct spdk_bdev *bdev)
|
|
{
|
|
return bdev->write_cache;
|
|
}
|
|
|
|
const struct spdk_uuid *
|
|
spdk_bdev_get_uuid(const struct spdk_bdev *bdev)
|
|
{
|
|
return &bdev->uuid;
|
|
}
|
|
|
|
uint32_t
|
|
spdk_bdev_get_md_size(const struct spdk_bdev *bdev)
|
|
{
|
|
return bdev->md_len;
|
|
}
|
|
|
|
bool
|
|
spdk_bdev_is_md_interleaved(const struct spdk_bdev *bdev)
|
|
{
|
|
return (bdev->md_len != 0) && bdev->md_interleave;
|
|
}
|
|
|
|
uint32_t
|
|
spdk_bdev_get_data_block_size(const struct spdk_bdev *bdev)
|
|
{
|
|
if (spdk_bdev_is_md_interleaved(bdev)) {
|
|
return bdev->blocklen - bdev->md_len;
|
|
} else {
|
|
return bdev->blocklen;
|
|
}
|
|
}
|
|
|
|
enum spdk_dif_type spdk_bdev_get_dif_type(const struct spdk_bdev *bdev)
|
|
{
|
|
if (bdev->md_len != 0) {
|
|
return bdev->dif_type;
|
|
} else {
|
|
return SPDK_DIF_DISABLE;
|
|
}
|
|
}
|
|
|
|
bool
|
|
spdk_bdev_is_dif_head_of_md(const struct spdk_bdev *bdev)
|
|
{
|
|
if (spdk_bdev_get_dif_type(bdev) != SPDK_DIF_DISABLE) {
|
|
return bdev->dif_is_head_of_md;
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
bool
|
|
spdk_bdev_is_dif_check_enabled(const struct spdk_bdev *bdev,
|
|
enum spdk_dif_check_type check_type)
|
|
{
|
|
if (spdk_bdev_get_dif_type(bdev) == SPDK_DIF_DISABLE) {
|
|
return false;
|
|
}
|
|
|
|
switch (check_type) {
|
|
case SPDK_DIF_CHECK_TYPE_REFTAG:
|
|
return (bdev->dif_check_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) != 0;
|
|
case SPDK_DIF_CHECK_TYPE_APPTAG:
|
|
return (bdev->dif_check_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) != 0;
|
|
case SPDK_DIF_CHECK_TYPE_GUARD:
|
|
return (bdev->dif_check_flags & SPDK_DIF_FLAGS_GUARD_CHECK) != 0;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
uint64_t
|
|
spdk_bdev_get_qd(const struct spdk_bdev *bdev)
|
|
{
|
|
return bdev->internal.measured_queue_depth;
|
|
}
|
|
|
|
uint64_t
|
|
spdk_bdev_get_qd_sampling_period(const struct spdk_bdev *bdev)
|
|
{
|
|
return bdev->internal.period;
|
|
}
|
|
|
|
uint64_t
|
|
spdk_bdev_get_weighted_io_time(const struct spdk_bdev *bdev)
|
|
{
|
|
return bdev->internal.weighted_io_time;
|
|
}
|
|
|
|
uint64_t
|
|
spdk_bdev_get_io_time(const struct spdk_bdev *bdev)
|
|
{
|
|
return bdev->internal.io_time;
|
|
}
|
|
|
|
static void
|
|
_calculate_measured_qd_cpl(struct spdk_io_channel_iter *i, int status)
|
|
{
|
|
struct spdk_bdev *bdev = spdk_io_channel_iter_get_ctx(i);
|
|
|
|
bdev->internal.measured_queue_depth = bdev->internal.temporary_queue_depth;
|
|
|
|
if (bdev->internal.measured_queue_depth) {
|
|
bdev->internal.io_time += bdev->internal.period;
|
|
bdev->internal.weighted_io_time += bdev->internal.period * bdev->internal.measured_queue_depth;
|
|
}
|
|
}
|
|
|
|
static void
|
|
_calculate_measured_qd(struct spdk_io_channel_iter *i)
|
|
{
|
|
struct spdk_bdev *bdev = spdk_io_channel_iter_get_ctx(i);
|
|
struct spdk_io_channel *io_ch = spdk_io_channel_iter_get_channel(i);
|
|
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(io_ch);
|
|
|
|
bdev->internal.temporary_queue_depth += ch->io_outstanding;
|
|
spdk_for_each_channel_continue(i, 0);
|
|
}
|
|
|
|
static int
|
|
spdk_bdev_calculate_measured_queue_depth(void *ctx)
|
|
{
|
|
struct spdk_bdev *bdev = ctx;
|
|
bdev->internal.temporary_queue_depth = 0;
|
|
spdk_for_each_channel(__bdev_to_io_dev(bdev), _calculate_measured_qd, bdev,
|
|
_calculate_measured_qd_cpl);
|
|
return 0;
|
|
}
|
|
|
|
void
|
|
spdk_bdev_set_qd_sampling_period(struct spdk_bdev *bdev, uint64_t period)
|
|
{
|
|
bdev->internal.period = period;
|
|
|
|
if (bdev->internal.qd_poller != NULL) {
|
|
spdk_poller_unregister(&bdev->internal.qd_poller);
|
|
bdev->internal.measured_queue_depth = UINT64_MAX;
|
|
}
|
|
|
|
if (period != 0) {
|
|
bdev->internal.qd_poller = spdk_poller_register(spdk_bdev_calculate_measured_queue_depth, bdev,
|
|
period);
|
|
}
|
|
}
|
|
|
|
int
|
|
spdk_bdev_notify_blockcnt_change(struct spdk_bdev *bdev, uint64_t size)
|
|
{
|
|
int ret;
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
|
|
/* bdev has open descriptors */
|
|
if (!TAILQ_EMPTY(&bdev->internal.open_descs) &&
|
|
bdev->blockcnt > size) {
|
|
ret = -EBUSY;
|
|
} else {
|
|
bdev->blockcnt = size;
|
|
ret = 0;
|
|
}
|
|
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
|
|
return ret;
|
|
}
|
|
|
|
/*
|
|
* Convert I/O offset and length from bytes to blocks.
|
|
*
|
|
* Returns zero on success or non-zero if the byte parameters aren't divisible by the block size.
|
|
*/
|
|
static uint64_t
|
|
spdk_bdev_bytes_to_blocks(struct spdk_bdev *bdev, uint64_t offset_bytes, uint64_t *offset_blocks,
|
|
uint64_t num_bytes, uint64_t *num_blocks)
|
|
{
|
|
uint32_t block_size = bdev->blocklen;
|
|
uint8_t shift_cnt;
|
|
|
|
/* Avoid expensive div operations if possible. These spdk_u32 functions are very cheap. */
|
|
if (spdk_likely(spdk_u32_is_pow2(block_size))) {
|
|
shift_cnt = spdk_u32log2(block_size);
|
|
*offset_blocks = offset_bytes >> shift_cnt;
|
|
*num_blocks = num_bytes >> shift_cnt;
|
|
return (offset_bytes - (*offset_blocks << shift_cnt)) |
|
|
(num_bytes - (*num_blocks << shift_cnt));
|
|
} else {
|
|
*offset_blocks = offset_bytes / block_size;
|
|
*num_blocks = num_bytes / block_size;
|
|
return (offset_bytes % block_size) | (num_bytes % block_size);
|
|
}
|
|
}
|
|
|
|
static bool
|
|
spdk_bdev_io_valid_blocks(struct spdk_bdev *bdev, uint64_t offset_blocks, uint64_t num_blocks)
|
|
{
|
|
/* Return failure if offset_blocks + num_blocks is less than offset_blocks; indicates there
|
|
* has been an overflow and hence the offset has been wrapped around */
|
|
if (offset_blocks + num_blocks < offset_blocks) {
|
|
return false;
|
|
}
|
|
|
|
/* Return failure if offset_blocks + num_blocks exceeds the size of the bdev */
|
|
if (offset_blocks + num_blocks > bdev->blockcnt) {
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
int
|
|
spdk_bdev_read(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
void *buf, uint64_t offset, uint64_t nbytes,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
uint64_t offset_blocks, num_blocks;
|
|
|
|
if (spdk_bdev_bytes_to_blocks(desc->bdev, offset, &offset_blocks, nbytes, &num_blocks) != 0) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
return spdk_bdev_read_blocks(desc, ch, buf, offset_blocks, num_blocks, cb, cb_arg);
|
|
}
|
|
|
|
int
|
|
spdk_bdev_read_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
void *buf, uint64_t offset_blocks, uint64_t num_blocks,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
struct spdk_bdev *bdev = desc->bdev;
|
|
struct spdk_bdev_io *bdev_io;
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
|
|
if (!spdk_bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
bdev_io = spdk_bdev_get_io(channel);
|
|
if (!bdev_io) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
bdev_io->internal.ch = channel;
|
|
bdev_io->internal.desc = desc;
|
|
bdev_io->type = SPDK_BDEV_IO_TYPE_READ;
|
|
bdev_io->u.bdev.iovs = &bdev_io->iov;
|
|
bdev_io->u.bdev.iovs[0].iov_base = buf;
|
|
bdev_io->u.bdev.iovs[0].iov_len = num_blocks * bdev->blocklen;
|
|
bdev_io->u.bdev.iovcnt = 1;
|
|
bdev_io->u.bdev.num_blocks = num_blocks;
|
|
bdev_io->u.bdev.offset_blocks = offset_blocks;
|
|
spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb);
|
|
|
|
spdk_bdev_io_submit(bdev_io);
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
spdk_bdev_readv(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
struct iovec *iov, int iovcnt,
|
|
uint64_t offset, uint64_t nbytes,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
uint64_t offset_blocks, num_blocks;
|
|
|
|
if (spdk_bdev_bytes_to_blocks(desc->bdev, offset, &offset_blocks, nbytes, &num_blocks) != 0) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
return spdk_bdev_readv_blocks(desc, ch, iov, iovcnt, offset_blocks, num_blocks, cb, cb_arg);
|
|
}
|
|
|
|
int spdk_bdev_readv_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
struct iovec *iov, int iovcnt,
|
|
uint64_t offset_blocks, uint64_t num_blocks,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
struct spdk_bdev *bdev = desc->bdev;
|
|
struct spdk_bdev_io *bdev_io;
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
|
|
if (!spdk_bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
bdev_io = spdk_bdev_get_io(channel);
|
|
if (!bdev_io) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
bdev_io->internal.ch = channel;
|
|
bdev_io->internal.desc = desc;
|
|
bdev_io->type = SPDK_BDEV_IO_TYPE_READ;
|
|
bdev_io->u.bdev.iovs = iov;
|
|
bdev_io->u.bdev.iovcnt = iovcnt;
|
|
bdev_io->u.bdev.num_blocks = num_blocks;
|
|
bdev_io->u.bdev.offset_blocks = offset_blocks;
|
|
spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb);
|
|
|
|
spdk_bdev_io_submit(bdev_io);
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
spdk_bdev_write(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
void *buf, uint64_t offset, uint64_t nbytes,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
uint64_t offset_blocks, num_blocks;
|
|
|
|
if (spdk_bdev_bytes_to_blocks(desc->bdev, offset, &offset_blocks, nbytes, &num_blocks) != 0) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
return spdk_bdev_write_blocks(desc, ch, buf, offset_blocks, num_blocks, cb, cb_arg);
|
|
}
|
|
|
|
int
|
|
spdk_bdev_write_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
void *buf, uint64_t offset_blocks, uint64_t num_blocks,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
struct spdk_bdev *bdev = desc->bdev;
|
|
struct spdk_bdev_io *bdev_io;
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
|
|
if (!desc->write) {
|
|
return -EBADF;
|
|
}
|
|
|
|
if (!spdk_bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
bdev_io = spdk_bdev_get_io(channel);
|
|
if (!bdev_io) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
bdev_io->internal.ch = channel;
|
|
bdev_io->internal.desc = desc;
|
|
bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE;
|
|
bdev_io->u.bdev.iovs = &bdev_io->iov;
|
|
bdev_io->u.bdev.iovs[0].iov_base = buf;
|
|
bdev_io->u.bdev.iovs[0].iov_len = num_blocks * bdev->blocklen;
|
|
bdev_io->u.bdev.iovcnt = 1;
|
|
bdev_io->u.bdev.num_blocks = num_blocks;
|
|
bdev_io->u.bdev.offset_blocks = offset_blocks;
|
|
spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb);
|
|
|
|
spdk_bdev_io_submit(bdev_io);
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
spdk_bdev_writev(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
struct iovec *iov, int iovcnt,
|
|
uint64_t offset, uint64_t len,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
uint64_t offset_blocks, num_blocks;
|
|
|
|
if (spdk_bdev_bytes_to_blocks(desc->bdev, offset, &offset_blocks, len, &num_blocks) != 0) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
return spdk_bdev_writev_blocks(desc, ch, iov, iovcnt, offset_blocks, num_blocks, cb, cb_arg);
|
|
}
|
|
|
|
int
|
|
spdk_bdev_writev_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
struct iovec *iov, int iovcnt,
|
|
uint64_t offset_blocks, uint64_t num_blocks,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
struct spdk_bdev *bdev = desc->bdev;
|
|
struct spdk_bdev_io *bdev_io;
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
|
|
if (!desc->write) {
|
|
return -EBADF;
|
|
}
|
|
|
|
if (!spdk_bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
bdev_io = spdk_bdev_get_io(channel);
|
|
if (!bdev_io) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
bdev_io->internal.ch = channel;
|
|
bdev_io->internal.desc = desc;
|
|
bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE;
|
|
bdev_io->u.bdev.iovs = iov;
|
|
bdev_io->u.bdev.iovcnt = iovcnt;
|
|
bdev_io->u.bdev.num_blocks = num_blocks;
|
|
bdev_io->u.bdev.offset_blocks = offset_blocks;
|
|
spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb);
|
|
|
|
spdk_bdev_io_submit(bdev_io);
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
spdk_bdev_write_zeroes(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
uint64_t offset, uint64_t len,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
uint64_t offset_blocks, num_blocks;
|
|
|
|
if (spdk_bdev_bytes_to_blocks(desc->bdev, offset, &offset_blocks, len, &num_blocks) != 0) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
return spdk_bdev_write_zeroes_blocks(desc, ch, offset_blocks, num_blocks, cb, cb_arg);
|
|
}
|
|
|
|
int
|
|
spdk_bdev_write_zeroes_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
uint64_t offset_blocks, uint64_t num_blocks,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
struct spdk_bdev *bdev = desc->bdev;
|
|
struct spdk_bdev_io *bdev_io;
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
|
|
if (!desc->write) {
|
|
return -EBADF;
|
|
}
|
|
|
|
if (!spdk_bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
bdev_io = spdk_bdev_get_io(channel);
|
|
|
|
if (!bdev_io) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE_ZEROES;
|
|
bdev_io->internal.ch = channel;
|
|
bdev_io->internal.desc = desc;
|
|
bdev_io->u.bdev.offset_blocks = offset_blocks;
|
|
bdev_io->u.bdev.num_blocks = num_blocks;
|
|
spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb);
|
|
|
|
if (_spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_WRITE_ZEROES)) {
|
|
spdk_bdev_io_submit(bdev_io);
|
|
return 0;
|
|
} else if (_spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_WRITE)) {
|
|
assert(spdk_bdev_get_block_size(bdev) <= ZERO_BUFFER_SIZE);
|
|
bdev_io->u.bdev.split_remaining_num_blocks = num_blocks;
|
|
bdev_io->u.bdev.split_current_offset_blocks = offset_blocks;
|
|
_spdk_bdev_write_zero_buffer_next(bdev_io);
|
|
return 0;
|
|
} else {
|
|
spdk_bdev_free_io(bdev_io);
|
|
return -ENOTSUP;
|
|
}
|
|
}
|
|
|
|
int
|
|
spdk_bdev_unmap(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
uint64_t offset, uint64_t nbytes,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
uint64_t offset_blocks, num_blocks;
|
|
|
|
if (spdk_bdev_bytes_to_blocks(desc->bdev, offset, &offset_blocks, nbytes, &num_blocks) != 0) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
return spdk_bdev_unmap_blocks(desc, ch, offset_blocks, num_blocks, cb, cb_arg);
|
|
}
|
|
|
|
int
|
|
spdk_bdev_unmap_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
uint64_t offset_blocks, uint64_t num_blocks,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
struct spdk_bdev *bdev = desc->bdev;
|
|
struct spdk_bdev_io *bdev_io;
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
|
|
if (!desc->write) {
|
|
return -EBADF;
|
|
}
|
|
|
|
if (!spdk_bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (num_blocks == 0) {
|
|
SPDK_ERRLOG("Can't unmap 0 bytes\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
bdev_io = spdk_bdev_get_io(channel);
|
|
if (!bdev_io) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
bdev_io->internal.ch = channel;
|
|
bdev_io->internal.desc = desc;
|
|
bdev_io->type = SPDK_BDEV_IO_TYPE_UNMAP;
|
|
|
|
bdev_io->u.bdev.iovs = &bdev_io->iov;
|
|
bdev_io->u.bdev.iovs[0].iov_base = NULL;
|
|
bdev_io->u.bdev.iovs[0].iov_len = 0;
|
|
bdev_io->u.bdev.iovcnt = 1;
|
|
|
|
bdev_io->u.bdev.offset_blocks = offset_blocks;
|
|
bdev_io->u.bdev.num_blocks = num_blocks;
|
|
spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb);
|
|
|
|
spdk_bdev_io_submit(bdev_io);
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
spdk_bdev_flush(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
uint64_t offset, uint64_t length,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
uint64_t offset_blocks, num_blocks;
|
|
|
|
if (spdk_bdev_bytes_to_blocks(desc->bdev, offset, &offset_blocks, length, &num_blocks) != 0) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
return spdk_bdev_flush_blocks(desc, ch, offset_blocks, num_blocks, cb, cb_arg);
|
|
}
|
|
|
|
int
|
|
spdk_bdev_flush_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
uint64_t offset_blocks, uint64_t num_blocks,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
struct spdk_bdev *bdev = desc->bdev;
|
|
struct spdk_bdev_io *bdev_io;
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
|
|
if (!desc->write) {
|
|
return -EBADF;
|
|
}
|
|
|
|
if (!spdk_bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
bdev_io = spdk_bdev_get_io(channel);
|
|
if (!bdev_io) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
bdev_io->internal.ch = channel;
|
|
bdev_io->internal.desc = desc;
|
|
bdev_io->type = SPDK_BDEV_IO_TYPE_FLUSH;
|
|
bdev_io->u.bdev.iovs = NULL;
|
|
bdev_io->u.bdev.iovcnt = 0;
|
|
bdev_io->u.bdev.offset_blocks = offset_blocks;
|
|
bdev_io->u.bdev.num_blocks = num_blocks;
|
|
spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb);
|
|
|
|
spdk_bdev_io_submit(bdev_io);
|
|
return 0;
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_reset_dev(struct spdk_io_channel_iter *i, int status)
|
|
{
|
|
struct spdk_bdev_channel *ch = spdk_io_channel_iter_get_ctx(i);
|
|
struct spdk_bdev_io *bdev_io;
|
|
|
|
bdev_io = TAILQ_FIRST(&ch->queued_resets);
|
|
TAILQ_REMOVE(&ch->queued_resets, bdev_io, internal.link);
|
|
spdk_bdev_io_submit_reset(bdev_io);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_reset_freeze_channel(struct spdk_io_channel_iter *i)
|
|
{
|
|
struct spdk_io_channel *ch;
|
|
struct spdk_bdev_channel *channel;
|
|
struct spdk_bdev_mgmt_channel *mgmt_channel;
|
|
struct spdk_bdev_shared_resource *shared_resource;
|
|
bdev_io_tailq_t tmp_queued;
|
|
|
|
TAILQ_INIT(&tmp_queued);
|
|
|
|
ch = spdk_io_channel_iter_get_channel(i);
|
|
channel = spdk_io_channel_get_ctx(ch);
|
|
shared_resource = channel->shared_resource;
|
|
mgmt_channel = shared_resource->mgmt_ch;
|
|
|
|
channel->flags |= BDEV_CH_RESET_IN_PROGRESS;
|
|
|
|
if ((channel->flags & BDEV_CH_QOS_ENABLED) != 0) {
|
|
/* The QoS object is always valid and readable while
|
|
* the channel flag is set, so the lock here should not
|
|
* be necessary. We're not in the fast path though, so
|
|
* just take it anyway. */
|
|
pthread_mutex_lock(&channel->bdev->internal.mutex);
|
|
if (channel->bdev->internal.qos->ch == channel) {
|
|
TAILQ_SWAP(&channel->bdev->internal.qos->queued, &tmp_queued, spdk_bdev_io, internal.link);
|
|
}
|
|
pthread_mutex_unlock(&channel->bdev->internal.mutex);
|
|
}
|
|
|
|
_spdk_bdev_abort_queued_io(&shared_resource->nomem_io, channel);
|
|
_spdk_bdev_abort_buf_io(&mgmt_channel->need_buf_small, channel);
|
|
_spdk_bdev_abort_buf_io(&mgmt_channel->need_buf_large, channel);
|
|
_spdk_bdev_abort_queued_io(&tmp_queued, channel);
|
|
|
|
spdk_for_each_channel_continue(i, 0);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_start_reset(void *ctx)
|
|
{
|
|
struct spdk_bdev_channel *ch = ctx;
|
|
|
|
spdk_for_each_channel(__bdev_to_io_dev(ch->bdev), _spdk_bdev_reset_freeze_channel,
|
|
ch, _spdk_bdev_reset_dev);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_channel_start_reset(struct spdk_bdev_channel *ch)
|
|
{
|
|
struct spdk_bdev *bdev = ch->bdev;
|
|
|
|
assert(!TAILQ_EMPTY(&ch->queued_resets));
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
if (bdev->internal.reset_in_progress == NULL) {
|
|
bdev->internal.reset_in_progress = TAILQ_FIRST(&ch->queued_resets);
|
|
/*
|
|
* Take a channel reference for the target bdev for the life of this
|
|
* reset. This guards against the channel getting destroyed while
|
|
* spdk_for_each_channel() calls related to this reset IO are in
|
|
* progress. We will release the reference when this reset is
|
|
* completed.
|
|
*/
|
|
bdev->internal.reset_in_progress->u.reset.ch_ref = spdk_get_io_channel(__bdev_to_io_dev(bdev));
|
|
_spdk_bdev_start_reset(ch);
|
|
}
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
}
|
|
|
|
int
|
|
spdk_bdev_reset(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
struct spdk_bdev *bdev = desc->bdev;
|
|
struct spdk_bdev_io *bdev_io;
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
|
|
bdev_io = spdk_bdev_get_io(channel);
|
|
if (!bdev_io) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
bdev_io->internal.ch = channel;
|
|
bdev_io->internal.desc = desc;
|
|
bdev_io->type = SPDK_BDEV_IO_TYPE_RESET;
|
|
bdev_io->u.reset.ch_ref = NULL;
|
|
spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb);
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
TAILQ_INSERT_TAIL(&channel->queued_resets, bdev_io, internal.link);
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
|
|
_spdk_bdev_channel_start_reset(channel);
|
|
|
|
return 0;
|
|
}
|
|
|
|
void
|
|
spdk_bdev_get_io_stat(struct spdk_bdev *bdev, struct spdk_io_channel *ch,
|
|
struct spdk_bdev_io_stat *stat)
|
|
{
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
|
|
*stat = channel->stat;
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_get_device_stat_done(struct spdk_io_channel_iter *i, int status)
|
|
{
|
|
void *io_device = spdk_io_channel_iter_get_io_device(i);
|
|
struct spdk_bdev_iostat_ctx *bdev_iostat_ctx = spdk_io_channel_iter_get_ctx(i);
|
|
|
|
bdev_iostat_ctx->cb(__bdev_from_io_dev(io_device), bdev_iostat_ctx->stat,
|
|
bdev_iostat_ctx->cb_arg, 0);
|
|
free(bdev_iostat_ctx);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_get_each_channel_stat(struct spdk_io_channel_iter *i)
|
|
{
|
|
struct spdk_bdev_iostat_ctx *bdev_iostat_ctx = spdk_io_channel_iter_get_ctx(i);
|
|
struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
|
|
_spdk_bdev_io_stat_add(bdev_iostat_ctx->stat, &channel->stat);
|
|
spdk_for_each_channel_continue(i, 0);
|
|
}
|
|
|
|
void
|
|
spdk_bdev_get_device_stat(struct spdk_bdev *bdev, struct spdk_bdev_io_stat *stat,
|
|
spdk_bdev_get_device_stat_cb cb, void *cb_arg)
|
|
{
|
|
struct spdk_bdev_iostat_ctx *bdev_iostat_ctx;
|
|
|
|
assert(bdev != NULL);
|
|
assert(stat != NULL);
|
|
assert(cb != NULL);
|
|
|
|
bdev_iostat_ctx = calloc(1, sizeof(struct spdk_bdev_iostat_ctx));
|
|
if (bdev_iostat_ctx == NULL) {
|
|
SPDK_ERRLOG("Unable to allocate memory for spdk_bdev_iostat_ctx\n");
|
|
cb(bdev, stat, cb_arg, -ENOMEM);
|
|
return;
|
|
}
|
|
|
|
bdev_iostat_ctx->stat = stat;
|
|
bdev_iostat_ctx->cb = cb;
|
|
bdev_iostat_ctx->cb_arg = cb_arg;
|
|
|
|
/* Start with the statistics from previously deleted channels. */
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
_spdk_bdev_io_stat_add(bdev_iostat_ctx->stat, &bdev->internal.stat);
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
|
|
/* Then iterate and add the statistics from each existing channel. */
|
|
spdk_for_each_channel(__bdev_to_io_dev(bdev),
|
|
_spdk_bdev_get_each_channel_stat,
|
|
bdev_iostat_ctx,
|
|
_spdk_bdev_get_device_stat_done);
|
|
}
|
|
|
|
int
|
|
spdk_bdev_nvme_admin_passthru(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
const struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
struct spdk_bdev *bdev = desc->bdev;
|
|
struct spdk_bdev_io *bdev_io;
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
|
|
if (!desc->write) {
|
|
return -EBADF;
|
|
}
|
|
|
|
bdev_io = spdk_bdev_get_io(channel);
|
|
if (!bdev_io) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
bdev_io->internal.ch = channel;
|
|
bdev_io->internal.desc = desc;
|
|
bdev_io->type = SPDK_BDEV_IO_TYPE_NVME_ADMIN;
|
|
bdev_io->u.nvme_passthru.cmd = *cmd;
|
|
bdev_io->u.nvme_passthru.buf = buf;
|
|
bdev_io->u.nvme_passthru.nbytes = nbytes;
|
|
bdev_io->u.nvme_passthru.md_buf = NULL;
|
|
bdev_io->u.nvme_passthru.md_len = 0;
|
|
|
|
spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb);
|
|
|
|
spdk_bdev_io_submit(bdev_io);
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
spdk_bdev_nvme_io_passthru(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
const struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
struct spdk_bdev *bdev = desc->bdev;
|
|
struct spdk_bdev_io *bdev_io;
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
|
|
if (!desc->write) {
|
|
/*
|
|
* Do not try to parse the NVMe command - we could maybe use bits in the opcode
|
|
* to easily determine if the command is a read or write, but for now just
|
|
* do not allow io_passthru with a read-only descriptor.
|
|
*/
|
|
return -EBADF;
|
|
}
|
|
|
|
bdev_io = spdk_bdev_get_io(channel);
|
|
if (!bdev_io) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
bdev_io->internal.ch = channel;
|
|
bdev_io->internal.desc = desc;
|
|
bdev_io->type = SPDK_BDEV_IO_TYPE_NVME_IO;
|
|
bdev_io->u.nvme_passthru.cmd = *cmd;
|
|
bdev_io->u.nvme_passthru.buf = buf;
|
|
bdev_io->u.nvme_passthru.nbytes = nbytes;
|
|
bdev_io->u.nvme_passthru.md_buf = NULL;
|
|
bdev_io->u.nvme_passthru.md_len = 0;
|
|
|
|
spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb);
|
|
|
|
spdk_bdev_io_submit(bdev_io);
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
spdk_bdev_nvme_io_passthru_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
|
|
const struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes, void *md_buf, size_t md_len,
|
|
spdk_bdev_io_completion_cb cb, void *cb_arg)
|
|
{
|
|
struct spdk_bdev *bdev = desc->bdev;
|
|
struct spdk_bdev_io *bdev_io;
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
|
|
if (!desc->write) {
|
|
/*
|
|
* Do not try to parse the NVMe command - we could maybe use bits in the opcode
|
|
* to easily determine if the command is a read or write, but for now just
|
|
* do not allow io_passthru with a read-only descriptor.
|
|
*/
|
|
return -EBADF;
|
|
}
|
|
|
|
bdev_io = spdk_bdev_get_io(channel);
|
|
if (!bdev_io) {
|
|
return -ENOMEM;
|
|
}
|
|
|
|
bdev_io->internal.ch = channel;
|
|
bdev_io->internal.desc = desc;
|
|
bdev_io->type = SPDK_BDEV_IO_TYPE_NVME_IO_MD;
|
|
bdev_io->u.nvme_passthru.cmd = *cmd;
|
|
bdev_io->u.nvme_passthru.buf = buf;
|
|
bdev_io->u.nvme_passthru.nbytes = nbytes;
|
|
bdev_io->u.nvme_passthru.md_buf = md_buf;
|
|
bdev_io->u.nvme_passthru.md_len = md_len;
|
|
|
|
spdk_bdev_io_init(bdev_io, bdev, cb_arg, cb);
|
|
|
|
spdk_bdev_io_submit(bdev_io);
|
|
return 0;
|
|
}
|
|
|
|
int
|
|
spdk_bdev_queue_io_wait(struct spdk_bdev *bdev, struct spdk_io_channel *ch,
|
|
struct spdk_bdev_io_wait_entry *entry)
|
|
{
|
|
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
|
|
struct spdk_bdev_mgmt_channel *mgmt_ch = channel->shared_resource->mgmt_ch;
|
|
|
|
if (bdev != entry->bdev) {
|
|
SPDK_ERRLOG("bdevs do not match\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (mgmt_ch->per_thread_cache_count > 0) {
|
|
SPDK_ERRLOG("Cannot queue io_wait if spdk_bdev_io available in per-thread cache\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
TAILQ_INSERT_TAIL(&mgmt_ch->io_wait_queue, entry, link);
|
|
return 0;
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_ch_retry_io(struct spdk_bdev_channel *bdev_ch)
|
|
{
|
|
struct spdk_bdev *bdev = bdev_ch->bdev;
|
|
struct spdk_bdev_shared_resource *shared_resource = bdev_ch->shared_resource;
|
|
struct spdk_bdev_io *bdev_io;
|
|
|
|
if (shared_resource->io_outstanding > shared_resource->nomem_threshold) {
|
|
/*
|
|
* Allow some more I/O to complete before retrying the nomem_io queue.
|
|
* Some drivers (such as nvme) cannot immediately take a new I/O in
|
|
* the context of a completion, because the resources for the I/O are
|
|
* not released until control returns to the bdev poller. Also, we
|
|
* may require several small I/O to complete before a larger I/O
|
|
* (that requires splitting) can be submitted.
|
|
*/
|
|
return;
|
|
}
|
|
|
|
while (!TAILQ_EMPTY(&shared_resource->nomem_io)) {
|
|
bdev_io = TAILQ_FIRST(&shared_resource->nomem_io);
|
|
TAILQ_REMOVE(&shared_resource->nomem_io, bdev_io, internal.link);
|
|
bdev_io->internal.ch->io_outstanding++;
|
|
shared_resource->io_outstanding++;
|
|
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_PENDING;
|
|
bdev->fn_table->submit_request(bdev_io->internal.ch->channel, bdev_io);
|
|
if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_NOMEM) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
static inline void
|
|
_spdk_bdev_io_complete(void *ctx)
|
|
{
|
|
struct spdk_bdev_io *bdev_io = ctx;
|
|
uint64_t tsc, tsc_diff;
|
|
|
|
if (spdk_unlikely(bdev_io->internal.in_submit_request || bdev_io->internal.io_submit_ch)) {
|
|
/*
|
|
* Send the completion to the thread that originally submitted the I/O,
|
|
* which may not be the current thread in the case of QoS.
|
|
*/
|
|
if (bdev_io->internal.io_submit_ch) {
|
|
bdev_io->internal.ch = bdev_io->internal.io_submit_ch;
|
|
bdev_io->internal.io_submit_ch = NULL;
|
|
}
|
|
|
|
/*
|
|
* Defer completion to avoid potential infinite recursion if the
|
|
* user's completion callback issues a new I/O.
|
|
*/
|
|
spdk_thread_send_msg(spdk_io_channel_get_thread(bdev_io->internal.ch->channel),
|
|
_spdk_bdev_io_complete, bdev_io);
|
|
return;
|
|
}
|
|
|
|
tsc = spdk_get_ticks();
|
|
tsc_diff = tsc - bdev_io->internal.submit_tsc;
|
|
spdk_trace_record_tsc(tsc, TRACE_BDEV_IO_DONE, 0, 0, (uintptr_t)bdev_io, 0);
|
|
|
|
if (bdev_io->internal.ch->histogram) {
|
|
spdk_histogram_data_tally(bdev_io->internal.ch->histogram, tsc_diff);
|
|
}
|
|
|
|
if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS) {
|
|
switch (bdev_io->type) {
|
|
case SPDK_BDEV_IO_TYPE_READ:
|
|
bdev_io->internal.ch->stat.bytes_read += bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
|
|
bdev_io->internal.ch->stat.num_read_ops++;
|
|
bdev_io->internal.ch->stat.read_latency_ticks += tsc_diff;
|
|
break;
|
|
case SPDK_BDEV_IO_TYPE_WRITE:
|
|
bdev_io->internal.ch->stat.bytes_written += bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
|
|
bdev_io->internal.ch->stat.num_write_ops++;
|
|
bdev_io->internal.ch->stat.write_latency_ticks += tsc_diff;
|
|
break;
|
|
case SPDK_BDEV_IO_TYPE_UNMAP:
|
|
bdev_io->internal.ch->stat.bytes_unmapped += bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
|
|
bdev_io->internal.ch->stat.num_unmap_ops++;
|
|
bdev_io->internal.ch->stat.unmap_latency_ticks += tsc_diff;
|
|
default:
|
|
break;
|
|
}
|
|
}
|
|
|
|
#ifdef SPDK_CONFIG_VTUNE
|
|
uint64_t now_tsc = spdk_get_ticks();
|
|
if (now_tsc > (bdev_io->internal.ch->start_tsc + bdev_io->internal.ch->interval_tsc)) {
|
|
uint64_t data[5];
|
|
|
|
data[0] = bdev_io->internal.ch->stat.num_read_ops - bdev_io->internal.ch->prev_stat.num_read_ops;
|
|
data[1] = bdev_io->internal.ch->stat.bytes_read - bdev_io->internal.ch->prev_stat.bytes_read;
|
|
data[2] = bdev_io->internal.ch->stat.num_write_ops - bdev_io->internal.ch->prev_stat.num_write_ops;
|
|
data[3] = bdev_io->internal.ch->stat.bytes_written - bdev_io->internal.ch->prev_stat.bytes_written;
|
|
data[4] = bdev_io->bdev->fn_table->get_spin_time ?
|
|
bdev_io->bdev->fn_table->get_spin_time(bdev_io->internal.ch->channel) : 0;
|
|
|
|
__itt_metadata_add(g_bdev_mgr.domain, __itt_null, bdev_io->internal.ch->handle,
|
|
__itt_metadata_u64, 5, data);
|
|
|
|
bdev_io->internal.ch->prev_stat = bdev_io->internal.ch->stat;
|
|
bdev_io->internal.ch->start_tsc = now_tsc;
|
|
}
|
|
#endif
|
|
|
|
assert(bdev_io->internal.cb != NULL);
|
|
assert(spdk_get_thread() == spdk_io_channel_get_thread(bdev_io->internal.ch->channel));
|
|
|
|
bdev_io->internal.cb(bdev_io, bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS,
|
|
bdev_io->internal.caller_ctx);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_reset_complete(struct spdk_io_channel_iter *i, int status)
|
|
{
|
|
struct spdk_bdev_io *bdev_io = spdk_io_channel_iter_get_ctx(i);
|
|
|
|
if (bdev_io->u.reset.ch_ref != NULL) {
|
|
spdk_put_io_channel(bdev_io->u.reset.ch_ref);
|
|
bdev_io->u.reset.ch_ref = NULL;
|
|
}
|
|
|
|
_spdk_bdev_io_complete(bdev_io);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_unfreeze_channel(struct spdk_io_channel_iter *i)
|
|
{
|
|
struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
|
|
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(_ch);
|
|
|
|
ch->flags &= ~BDEV_CH_RESET_IN_PROGRESS;
|
|
if (!TAILQ_EMPTY(&ch->queued_resets)) {
|
|
_spdk_bdev_channel_start_reset(ch);
|
|
}
|
|
|
|
spdk_for_each_channel_continue(i, 0);
|
|
}
|
|
|
|
void
|
|
spdk_bdev_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status)
|
|
{
|
|
struct spdk_bdev *bdev = bdev_io->bdev;
|
|
struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch;
|
|
struct spdk_bdev_shared_resource *shared_resource = bdev_ch->shared_resource;
|
|
|
|
bdev_io->internal.status = status;
|
|
|
|
if (spdk_unlikely(bdev_io->type == SPDK_BDEV_IO_TYPE_RESET)) {
|
|
bool unlock_channels = false;
|
|
|
|
if (status == SPDK_BDEV_IO_STATUS_NOMEM) {
|
|
SPDK_ERRLOG("NOMEM returned for reset\n");
|
|
}
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
if (bdev_io == bdev->internal.reset_in_progress) {
|
|
bdev->internal.reset_in_progress = NULL;
|
|
unlock_channels = true;
|
|
}
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
|
|
if (unlock_channels) {
|
|
spdk_for_each_channel(__bdev_to_io_dev(bdev), _spdk_bdev_unfreeze_channel,
|
|
bdev_io, _spdk_bdev_reset_complete);
|
|
return;
|
|
}
|
|
} else {
|
|
if (spdk_unlikely(bdev_io->internal.orig_iovcnt > 0)) {
|
|
_bdev_io_unset_bounce_buf(bdev_io);
|
|
}
|
|
|
|
assert(bdev_ch->io_outstanding > 0);
|
|
assert(shared_resource->io_outstanding > 0);
|
|
bdev_ch->io_outstanding--;
|
|
shared_resource->io_outstanding--;
|
|
|
|
if (spdk_unlikely(status == SPDK_BDEV_IO_STATUS_NOMEM)) {
|
|
assert(shared_resource->io_outstanding > 0);
|
|
TAILQ_INSERT_HEAD(&shared_resource->nomem_io, bdev_io, internal.link);
|
|
/*
|
|
* Wait for some of the outstanding I/O to complete before we
|
|
* retry any of the nomem_io. Normally we will wait for
|
|
* NOMEM_THRESHOLD_COUNT I/O to complete but for low queue
|
|
* depth channels we will instead wait for half to complete.
|
|
*/
|
|
shared_resource->nomem_threshold = spdk_max((int64_t)shared_resource->io_outstanding / 2,
|
|
(int64_t)shared_resource->io_outstanding - NOMEM_THRESHOLD_COUNT);
|
|
return;
|
|
}
|
|
|
|
if (spdk_unlikely(!TAILQ_EMPTY(&shared_resource->nomem_io))) {
|
|
_spdk_bdev_ch_retry_io(bdev_ch);
|
|
}
|
|
}
|
|
|
|
_spdk_bdev_io_complete(bdev_io);
|
|
}
|
|
|
|
void
|
|
spdk_bdev_io_complete_scsi_status(struct spdk_bdev_io *bdev_io, enum spdk_scsi_status sc,
|
|
enum spdk_scsi_sense sk, uint8_t asc, uint8_t ascq)
|
|
{
|
|
if (sc == SPDK_SCSI_STATUS_GOOD) {
|
|
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
|
|
} else {
|
|
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SCSI_ERROR;
|
|
bdev_io->internal.error.scsi.sc = sc;
|
|
bdev_io->internal.error.scsi.sk = sk;
|
|
bdev_io->internal.error.scsi.asc = asc;
|
|
bdev_io->internal.error.scsi.ascq = ascq;
|
|
}
|
|
|
|
spdk_bdev_io_complete(bdev_io, bdev_io->internal.status);
|
|
}
|
|
|
|
void
|
|
spdk_bdev_io_get_scsi_status(const struct spdk_bdev_io *bdev_io,
|
|
int *sc, int *sk, int *asc, int *ascq)
|
|
{
|
|
assert(sc != NULL);
|
|
assert(sk != NULL);
|
|
assert(asc != NULL);
|
|
assert(ascq != NULL);
|
|
|
|
switch (bdev_io->internal.status) {
|
|
case SPDK_BDEV_IO_STATUS_SUCCESS:
|
|
*sc = SPDK_SCSI_STATUS_GOOD;
|
|
*sk = SPDK_SCSI_SENSE_NO_SENSE;
|
|
*asc = SPDK_SCSI_ASC_NO_ADDITIONAL_SENSE;
|
|
*ascq = SPDK_SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
|
|
break;
|
|
case SPDK_BDEV_IO_STATUS_NVME_ERROR:
|
|
spdk_scsi_nvme_translate(bdev_io, sc, sk, asc, ascq);
|
|
break;
|
|
case SPDK_BDEV_IO_STATUS_SCSI_ERROR:
|
|
*sc = bdev_io->internal.error.scsi.sc;
|
|
*sk = bdev_io->internal.error.scsi.sk;
|
|
*asc = bdev_io->internal.error.scsi.asc;
|
|
*ascq = bdev_io->internal.error.scsi.ascq;
|
|
break;
|
|
default:
|
|
*sc = SPDK_SCSI_STATUS_CHECK_CONDITION;
|
|
*sk = SPDK_SCSI_SENSE_ABORTED_COMMAND;
|
|
*asc = SPDK_SCSI_ASC_NO_ADDITIONAL_SENSE;
|
|
*ascq = SPDK_SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
|
|
break;
|
|
}
|
|
}
|
|
|
|
void
|
|
spdk_bdev_io_complete_nvme_status(struct spdk_bdev_io *bdev_io, int sct, int sc)
|
|
{
|
|
if (sct == SPDK_NVME_SCT_GENERIC && sc == SPDK_NVME_SC_SUCCESS) {
|
|
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
|
|
} else {
|
|
bdev_io->internal.error.nvme.sct = sct;
|
|
bdev_io->internal.error.nvme.sc = sc;
|
|
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_NVME_ERROR;
|
|
}
|
|
|
|
spdk_bdev_io_complete(bdev_io, bdev_io->internal.status);
|
|
}
|
|
|
|
void
|
|
spdk_bdev_io_get_nvme_status(const struct spdk_bdev_io *bdev_io, int *sct, int *sc)
|
|
{
|
|
assert(sct != NULL);
|
|
assert(sc != NULL);
|
|
|
|
if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_NVME_ERROR) {
|
|
*sct = bdev_io->internal.error.nvme.sct;
|
|
*sc = bdev_io->internal.error.nvme.sc;
|
|
} else if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS) {
|
|
*sct = SPDK_NVME_SCT_GENERIC;
|
|
*sc = SPDK_NVME_SC_SUCCESS;
|
|
} else {
|
|
*sct = SPDK_NVME_SCT_GENERIC;
|
|
*sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
|
|
}
|
|
}
|
|
|
|
struct spdk_thread *
|
|
spdk_bdev_io_get_thread(struct spdk_bdev_io *bdev_io)
|
|
{
|
|
return spdk_io_channel_get_thread(bdev_io->internal.ch->channel);
|
|
}
|
|
|
|
struct spdk_io_channel *
|
|
spdk_bdev_io_get_io_channel(struct spdk_bdev_io *bdev_io)
|
|
{
|
|
return bdev_io->internal.ch->channel;
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_qos_config_limit(struct spdk_bdev *bdev, uint64_t *limits)
|
|
{
|
|
uint64_t min_qos_set;
|
|
int i;
|
|
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
if (limits[i] != SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (i == SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES) {
|
|
SPDK_ERRLOG("Invalid rate limits set.\n");
|
|
return;
|
|
}
|
|
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
if (limits[i] == SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {
|
|
continue;
|
|
}
|
|
|
|
if (_spdk_bdev_qos_is_iops_rate_limit(i) == true) {
|
|
min_qos_set = SPDK_BDEV_QOS_MIN_IOS_PER_SEC;
|
|
} else {
|
|
min_qos_set = SPDK_BDEV_QOS_MIN_BYTES_PER_SEC;
|
|
}
|
|
|
|
if (limits[i] == 0 || limits[i] % min_qos_set) {
|
|
SPDK_ERRLOG("Assigned limit %" PRIu64 " on bdev %s is not multiple of %" PRIu64 "\n",
|
|
limits[i], bdev->name, min_qos_set);
|
|
SPDK_ERRLOG("Failed to enable QoS on this bdev %s\n", bdev->name);
|
|
return;
|
|
}
|
|
}
|
|
|
|
if (!bdev->internal.qos) {
|
|
bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos));
|
|
if (!bdev->internal.qos) {
|
|
SPDK_ERRLOG("Unable to allocate memory for QoS tracking\n");
|
|
return;
|
|
}
|
|
}
|
|
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
bdev->internal.qos->rate_limits[i].limit = limits[i];
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Bdev:%s QoS type:%d set:%lu\n",
|
|
bdev->name, i, limits[i]);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_qos_config(struct spdk_bdev *bdev)
|
|
{
|
|
struct spdk_conf_section *sp = NULL;
|
|
const char *val = NULL;
|
|
int i = 0, j = 0;
|
|
uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES] = {};
|
|
bool config_qos = false;
|
|
|
|
sp = spdk_conf_find_section(NULL, "QoS");
|
|
if (!sp) {
|
|
return;
|
|
}
|
|
|
|
while (j < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES) {
|
|
limits[j] = SPDK_BDEV_QOS_LIMIT_NOT_DEFINED;
|
|
|
|
i = 0;
|
|
while (true) {
|
|
val = spdk_conf_section_get_nmval(sp, qos_conf_type[j], i, 0);
|
|
if (!val) {
|
|
break;
|
|
}
|
|
|
|
if (strcmp(bdev->name, val) != 0) {
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
val = spdk_conf_section_get_nmval(sp, qos_conf_type[j], i, 1);
|
|
if (val) {
|
|
if (_spdk_bdev_qos_is_iops_rate_limit(j) == true) {
|
|
limits[j] = strtoull(val, NULL, 10);
|
|
} else {
|
|
limits[j] = strtoull(val, NULL, 10) * 1024 * 1024;
|
|
}
|
|
config_qos = true;
|
|
}
|
|
|
|
break;
|
|
}
|
|
|
|
j++;
|
|
}
|
|
|
|
if (config_qos == true) {
|
|
_spdk_bdev_qos_config_limit(bdev, limits);
|
|
}
|
|
|
|
return;
|
|
}
|
|
|
|
static int
|
|
spdk_bdev_init(struct spdk_bdev *bdev)
|
|
{
|
|
char *bdev_name;
|
|
|
|
assert(bdev->module != NULL);
|
|
|
|
if (!bdev->name) {
|
|
SPDK_ERRLOG("Bdev name is NULL\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (spdk_bdev_get_by_name(bdev->name)) {
|
|
SPDK_ERRLOG("Bdev name:%s already exists\n", bdev->name);
|
|
return -EEXIST;
|
|
}
|
|
|
|
/* Users often register their own I/O devices using the bdev name. In
|
|
* order to avoid conflicts, prepend bdev_. */
|
|
bdev_name = spdk_sprintf_alloc("bdev_%s", bdev->name);
|
|
if (!bdev_name) {
|
|
SPDK_ERRLOG("Unable to allocate memory for internal bdev name.\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
bdev->internal.status = SPDK_BDEV_STATUS_READY;
|
|
bdev->internal.measured_queue_depth = UINT64_MAX;
|
|
bdev->internal.claim_module = NULL;
|
|
bdev->internal.qd_poller = NULL;
|
|
bdev->internal.qos = NULL;
|
|
|
|
if (spdk_bdev_get_buf_align(bdev) > 1) {
|
|
if (bdev->split_on_optimal_io_boundary) {
|
|
bdev->optimal_io_boundary = spdk_min(bdev->optimal_io_boundary,
|
|
SPDK_BDEV_LARGE_BUF_MAX_SIZE / bdev->blocklen);
|
|
} else {
|
|
bdev->split_on_optimal_io_boundary = true;
|
|
bdev->optimal_io_boundary = SPDK_BDEV_LARGE_BUF_MAX_SIZE / bdev->blocklen;
|
|
}
|
|
}
|
|
|
|
TAILQ_INIT(&bdev->internal.open_descs);
|
|
|
|
TAILQ_INIT(&bdev->aliases);
|
|
|
|
bdev->internal.reset_in_progress = NULL;
|
|
|
|
_spdk_bdev_qos_config(bdev);
|
|
|
|
spdk_io_device_register(__bdev_to_io_dev(bdev),
|
|
spdk_bdev_channel_create, spdk_bdev_channel_destroy,
|
|
sizeof(struct spdk_bdev_channel),
|
|
bdev_name);
|
|
|
|
free(bdev_name);
|
|
|
|
pthread_mutex_init(&bdev->internal.mutex, NULL);
|
|
return 0;
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_destroy_cb(void *io_device)
|
|
{
|
|
int rc;
|
|
struct spdk_bdev *bdev;
|
|
spdk_bdev_unregister_cb cb_fn;
|
|
void *cb_arg;
|
|
|
|
bdev = __bdev_from_io_dev(io_device);
|
|
cb_fn = bdev->internal.unregister_cb;
|
|
cb_arg = bdev->internal.unregister_ctx;
|
|
|
|
rc = bdev->fn_table->destruct(bdev->ctxt);
|
|
if (rc < 0) {
|
|
SPDK_ERRLOG("destruct failed\n");
|
|
}
|
|
if (rc <= 0 && cb_fn != NULL) {
|
|
cb_fn(cb_arg, rc);
|
|
}
|
|
}
|
|
|
|
|
|
static void
|
|
spdk_bdev_fini(struct spdk_bdev *bdev)
|
|
{
|
|
pthread_mutex_destroy(&bdev->internal.mutex);
|
|
|
|
free(bdev->internal.qos);
|
|
|
|
spdk_io_device_unregister(__bdev_to_io_dev(bdev), spdk_bdev_destroy_cb);
|
|
}
|
|
|
|
static void
|
|
spdk_bdev_start(struct spdk_bdev *bdev)
|
|
{
|
|
struct spdk_bdev_module *module;
|
|
uint32_t action;
|
|
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Inserting bdev %s into list\n", bdev->name);
|
|
TAILQ_INSERT_TAIL(&g_bdev_mgr.bdevs, bdev, internal.link);
|
|
|
|
/* Examine configuration before initializing I/O */
|
|
TAILQ_FOREACH(module, &g_bdev_mgr.bdev_modules, internal.tailq) {
|
|
if (module->examine_config) {
|
|
action = module->internal.action_in_progress;
|
|
module->internal.action_in_progress++;
|
|
module->examine_config(bdev);
|
|
if (action != module->internal.action_in_progress) {
|
|
SPDK_ERRLOG("examine_config for module %s did not call spdk_bdev_module_examine_done()\n",
|
|
module->name);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (bdev->internal.claim_module) {
|
|
return;
|
|
}
|
|
|
|
TAILQ_FOREACH(module, &g_bdev_mgr.bdev_modules, internal.tailq) {
|
|
if (module->examine_disk) {
|
|
module->internal.action_in_progress++;
|
|
module->examine_disk(bdev);
|
|
}
|
|
}
|
|
}
|
|
|
|
int
|
|
spdk_bdev_register(struct spdk_bdev *bdev)
|
|
{
|
|
int rc = spdk_bdev_init(bdev);
|
|
|
|
if (rc == 0) {
|
|
spdk_bdev_start(bdev);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
int
|
|
spdk_vbdev_register(struct spdk_bdev *vbdev, struct spdk_bdev **base_bdevs, int base_bdev_count)
|
|
{
|
|
int rc;
|
|
|
|
rc = spdk_bdev_init(vbdev);
|
|
if (rc) {
|
|
return rc;
|
|
}
|
|
|
|
spdk_bdev_start(vbdev);
|
|
return 0;
|
|
}
|
|
|
|
void
|
|
spdk_bdev_destruct_done(struct spdk_bdev *bdev, int bdeverrno)
|
|
{
|
|
if (bdev->internal.unregister_cb != NULL) {
|
|
bdev->internal.unregister_cb(bdev->internal.unregister_ctx, bdeverrno);
|
|
}
|
|
}
|
|
|
|
static void
|
|
_remove_notify(void *arg)
|
|
{
|
|
struct spdk_bdev_desc *desc = arg;
|
|
|
|
desc->remove_scheduled = false;
|
|
|
|
if (desc->closed) {
|
|
free(desc);
|
|
} else {
|
|
desc->remove_cb(desc->remove_ctx);
|
|
}
|
|
}
|
|
|
|
/* Must be called while holding bdev->internal.mutex.
|
|
* returns: 0 - bdev removed and ready to be destructed.
|
|
* -EBUSY - bdev can't be destructed yet. */
|
|
static int
|
|
spdk_bdev_unregister_unsafe(struct spdk_bdev *bdev)
|
|
{
|
|
struct spdk_bdev_desc *desc, *tmp;
|
|
int rc = 0;
|
|
|
|
TAILQ_FOREACH_SAFE(desc, &bdev->internal.open_descs, link, tmp) {
|
|
if (desc->remove_cb) {
|
|
rc = -EBUSY;
|
|
/*
|
|
* Defer invocation of the remove_cb to a separate message that will
|
|
* run later on its thread. This ensures this context unwinds and
|
|
* we don't recursively unregister this bdev again if the remove_cb
|
|
* immediately closes its descriptor.
|
|
*/
|
|
if (!desc->remove_scheduled) {
|
|
/* Avoid scheduling removal of the same descriptor multiple times. */
|
|
desc->remove_scheduled = true;
|
|
spdk_thread_send_msg(desc->thread, _remove_notify, desc);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (rc == 0) {
|
|
TAILQ_REMOVE(&g_bdev_mgr.bdevs, bdev, internal.link);
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Removing bdev %s from list done\n", bdev->name);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
void
|
|
spdk_bdev_unregister(struct spdk_bdev *bdev, spdk_bdev_unregister_cb cb_fn, void *cb_arg)
|
|
{
|
|
struct spdk_thread *thread;
|
|
int rc;
|
|
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Removing bdev %s from list\n", bdev->name);
|
|
|
|
thread = spdk_get_thread();
|
|
if (!thread) {
|
|
/* The user called this from a non-SPDK thread. */
|
|
if (cb_fn != NULL) {
|
|
cb_fn(cb_arg, -ENOTSUP);
|
|
}
|
|
return;
|
|
}
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
if (bdev->internal.status == SPDK_BDEV_STATUS_REMOVING) {
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
if (cb_fn) {
|
|
cb_fn(cb_arg, -EBUSY);
|
|
}
|
|
return;
|
|
}
|
|
|
|
bdev->internal.status = SPDK_BDEV_STATUS_REMOVING;
|
|
bdev->internal.unregister_cb = cb_fn;
|
|
bdev->internal.unregister_ctx = cb_arg;
|
|
|
|
/* Call under lock. */
|
|
rc = spdk_bdev_unregister_unsafe(bdev);
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
|
|
if (rc == 0) {
|
|
spdk_bdev_fini(bdev);
|
|
}
|
|
}
|
|
|
|
int
|
|
spdk_bdev_open(struct spdk_bdev *bdev, bool write, spdk_bdev_remove_cb_t remove_cb,
|
|
void *remove_ctx, struct spdk_bdev_desc **_desc)
|
|
{
|
|
struct spdk_bdev_desc *desc;
|
|
struct spdk_thread *thread;
|
|
struct set_qos_limit_ctx *ctx;
|
|
|
|
thread = spdk_get_thread();
|
|
if (!thread) {
|
|
SPDK_ERRLOG("Cannot open bdev from non-SPDK thread.\n");
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
desc = calloc(1, sizeof(*desc));
|
|
if (desc == NULL) {
|
|
SPDK_ERRLOG("Failed to allocate memory for bdev descriptor\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Opening descriptor %p for bdev %s on thread %p\n", desc, bdev->name,
|
|
spdk_get_thread());
|
|
|
|
desc->bdev = bdev;
|
|
desc->thread = thread;
|
|
desc->remove_cb = remove_cb;
|
|
desc->remove_ctx = remove_ctx;
|
|
desc->write = write;
|
|
*_desc = desc;
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
|
|
if (write && bdev->internal.claim_module) {
|
|
SPDK_ERRLOG("Could not open %s - %s module already claimed it\n",
|
|
bdev->name, bdev->internal.claim_module->name);
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
free(desc);
|
|
*_desc = NULL;
|
|
return -EPERM;
|
|
}
|
|
|
|
/* Enable QoS */
|
|
if (bdev->internal.qos && bdev->internal.qos->thread == NULL) {
|
|
ctx = calloc(1, sizeof(*ctx));
|
|
if (ctx == NULL) {
|
|
SPDK_ERRLOG("Failed to allocate memory for QoS context\n");
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
free(desc);
|
|
*_desc = NULL;
|
|
return -ENOMEM;
|
|
}
|
|
ctx->bdev = bdev;
|
|
spdk_for_each_channel(__bdev_to_io_dev(bdev),
|
|
_spdk_bdev_enable_qos_msg, ctx,
|
|
_spdk_bdev_enable_qos_done);
|
|
}
|
|
|
|
TAILQ_INSERT_TAIL(&bdev->internal.open_descs, desc, link);
|
|
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
|
|
return 0;
|
|
}
|
|
|
|
void
|
|
spdk_bdev_close(struct spdk_bdev_desc *desc)
|
|
{
|
|
struct spdk_bdev *bdev = desc->bdev;
|
|
int rc;
|
|
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Closing descriptor %p for bdev %s on thread %p\n", desc, bdev->name,
|
|
spdk_get_thread());
|
|
|
|
assert(desc->thread == spdk_get_thread());
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
|
|
TAILQ_REMOVE(&bdev->internal.open_descs, desc, link);
|
|
|
|
desc->closed = true;
|
|
|
|
if (!desc->remove_scheduled) {
|
|
free(desc);
|
|
}
|
|
|
|
/* If no more descriptors, kill QoS channel */
|
|
if (bdev->internal.qos && TAILQ_EMPTY(&bdev->internal.open_descs)) {
|
|
SPDK_DEBUGLOG(SPDK_LOG_BDEV, "Closed last descriptor for bdev %s on thread %p. Stopping QoS.\n",
|
|
bdev->name, spdk_get_thread());
|
|
|
|
if (spdk_bdev_qos_destroy(bdev)) {
|
|
/* There isn't anything we can do to recover here. Just let the
|
|
* old QoS poller keep running. The QoS handling won't change
|
|
* cores when the user allocates a new channel, but it won't break. */
|
|
SPDK_ERRLOG("Unable to shut down QoS poller. It will continue running on the current thread.\n");
|
|
}
|
|
}
|
|
|
|
spdk_bdev_set_qd_sampling_period(bdev, 0);
|
|
|
|
if (bdev->internal.status == SPDK_BDEV_STATUS_REMOVING && TAILQ_EMPTY(&bdev->internal.open_descs)) {
|
|
rc = spdk_bdev_unregister_unsafe(bdev);
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
|
|
if (rc == 0) {
|
|
spdk_bdev_fini(bdev);
|
|
}
|
|
} else {
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
}
|
|
}
|
|
|
|
int
|
|
spdk_bdev_module_claim_bdev(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
|
|
struct spdk_bdev_module *module)
|
|
{
|
|
if (bdev->internal.claim_module != NULL) {
|
|
SPDK_ERRLOG("bdev %s already claimed by module %s\n", bdev->name,
|
|
bdev->internal.claim_module->name);
|
|
return -EPERM;
|
|
}
|
|
|
|
if (desc && !desc->write) {
|
|
desc->write = true;
|
|
}
|
|
|
|
bdev->internal.claim_module = module;
|
|
return 0;
|
|
}
|
|
|
|
void
|
|
spdk_bdev_module_release_bdev(struct spdk_bdev *bdev)
|
|
{
|
|
assert(bdev->internal.claim_module != NULL);
|
|
bdev->internal.claim_module = NULL;
|
|
}
|
|
|
|
struct spdk_bdev *
|
|
spdk_bdev_desc_get_bdev(struct spdk_bdev_desc *desc)
|
|
{
|
|
return desc->bdev;
|
|
}
|
|
|
|
void
|
|
spdk_bdev_io_get_iovec(struct spdk_bdev_io *bdev_io, struct iovec **iovp, int *iovcntp)
|
|
{
|
|
struct iovec *iovs;
|
|
int iovcnt;
|
|
|
|
if (bdev_io == NULL) {
|
|
return;
|
|
}
|
|
|
|
switch (bdev_io->type) {
|
|
case SPDK_BDEV_IO_TYPE_READ:
|
|
iovs = bdev_io->u.bdev.iovs;
|
|
iovcnt = bdev_io->u.bdev.iovcnt;
|
|
break;
|
|
case SPDK_BDEV_IO_TYPE_WRITE:
|
|
iovs = bdev_io->u.bdev.iovs;
|
|
iovcnt = bdev_io->u.bdev.iovcnt;
|
|
break;
|
|
default:
|
|
iovs = NULL;
|
|
iovcnt = 0;
|
|
break;
|
|
}
|
|
|
|
if (iovp) {
|
|
*iovp = iovs;
|
|
}
|
|
if (iovcntp) {
|
|
*iovcntp = iovcnt;
|
|
}
|
|
}
|
|
|
|
void
|
|
spdk_bdev_module_list_add(struct spdk_bdev_module *bdev_module)
|
|
{
|
|
|
|
if (spdk_bdev_module_list_find(bdev_module->name)) {
|
|
SPDK_ERRLOG("ERROR: module '%s' already registered.\n", bdev_module->name);
|
|
assert(false);
|
|
}
|
|
|
|
if (bdev_module->async_init) {
|
|
bdev_module->internal.action_in_progress = 1;
|
|
}
|
|
|
|
/*
|
|
* Modules with examine callbacks must be initialized first, so they are
|
|
* ready to handle examine callbacks from later modules that will
|
|
* register physical bdevs.
|
|
*/
|
|
if (bdev_module->examine_config != NULL || bdev_module->examine_disk != NULL) {
|
|
TAILQ_INSERT_HEAD(&g_bdev_mgr.bdev_modules, bdev_module, internal.tailq);
|
|
} else {
|
|
TAILQ_INSERT_TAIL(&g_bdev_mgr.bdev_modules, bdev_module, internal.tailq);
|
|
}
|
|
}
|
|
|
|
struct spdk_bdev_module *
|
|
spdk_bdev_module_list_find(const char *name)
|
|
{
|
|
struct spdk_bdev_module *bdev_module;
|
|
|
|
TAILQ_FOREACH(bdev_module, &g_bdev_mgr.bdev_modules, internal.tailq) {
|
|
if (strcmp(name, bdev_module->name) == 0) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
return bdev_module;
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_write_zero_buffer_next(void *_bdev_io)
|
|
{
|
|
struct spdk_bdev_io *bdev_io = _bdev_io;
|
|
uint64_t num_bytes, num_blocks;
|
|
int rc;
|
|
|
|
num_bytes = spdk_min(spdk_bdev_get_block_size(bdev_io->bdev) *
|
|
bdev_io->u.bdev.split_remaining_num_blocks,
|
|
ZERO_BUFFER_SIZE);
|
|
num_blocks = num_bytes / spdk_bdev_get_block_size(bdev_io->bdev);
|
|
|
|
rc = spdk_bdev_write_blocks(bdev_io->internal.desc,
|
|
spdk_io_channel_from_ctx(bdev_io->internal.ch),
|
|
g_bdev_mgr.zero_buffer,
|
|
bdev_io->u.bdev.split_current_offset_blocks, num_blocks,
|
|
_spdk_bdev_write_zero_buffer_done, bdev_io);
|
|
if (rc == 0) {
|
|
bdev_io->u.bdev.split_remaining_num_blocks -= num_blocks;
|
|
bdev_io->u.bdev.split_current_offset_blocks += num_blocks;
|
|
} else if (rc == -ENOMEM) {
|
|
_spdk_bdev_queue_io_wait_with_cb(bdev_io, _spdk_bdev_write_zero_buffer_next);
|
|
} else {
|
|
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
|
|
bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
|
|
}
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_write_zero_buffer_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
|
|
{
|
|
struct spdk_bdev_io *parent_io = cb_arg;
|
|
|
|
spdk_bdev_free_io(bdev_io);
|
|
|
|
if (!success) {
|
|
parent_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
|
|
parent_io->internal.cb(parent_io, false, parent_io->internal.caller_ctx);
|
|
return;
|
|
}
|
|
|
|
if (parent_io->u.bdev.split_remaining_num_blocks == 0) {
|
|
parent_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
|
|
parent_io->internal.cb(parent_io, true, parent_io->internal.caller_ctx);
|
|
return;
|
|
}
|
|
|
|
_spdk_bdev_write_zero_buffer_next(parent_io);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_set_qos_limit_done(struct set_qos_limit_ctx *ctx, int status)
|
|
{
|
|
pthread_mutex_lock(&ctx->bdev->internal.mutex);
|
|
ctx->bdev->internal.qos_mod_in_progress = false;
|
|
pthread_mutex_unlock(&ctx->bdev->internal.mutex);
|
|
|
|
if (ctx->cb_fn) {
|
|
ctx->cb_fn(ctx->cb_arg, status);
|
|
}
|
|
free(ctx);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_disable_qos_done(void *cb_arg)
|
|
{
|
|
struct set_qos_limit_ctx *ctx = cb_arg;
|
|
struct spdk_bdev *bdev = ctx->bdev;
|
|
struct spdk_bdev_io *bdev_io;
|
|
struct spdk_bdev_qos *qos;
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
qos = bdev->internal.qos;
|
|
bdev->internal.qos = NULL;
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
|
|
while (!TAILQ_EMPTY(&qos->queued)) {
|
|
/* Send queued I/O back to their original thread for resubmission. */
|
|
bdev_io = TAILQ_FIRST(&qos->queued);
|
|
TAILQ_REMOVE(&qos->queued, bdev_io, internal.link);
|
|
|
|
if (bdev_io->internal.io_submit_ch) {
|
|
/*
|
|
* Channel was changed when sending it to the QoS thread - change it back
|
|
* before sending it back to the original thread.
|
|
*/
|
|
bdev_io->internal.ch = bdev_io->internal.io_submit_ch;
|
|
bdev_io->internal.io_submit_ch = NULL;
|
|
}
|
|
|
|
spdk_thread_send_msg(spdk_io_channel_get_thread(bdev_io->internal.ch->channel),
|
|
_spdk_bdev_io_submit, bdev_io);
|
|
}
|
|
|
|
if (qos->thread != NULL) {
|
|
spdk_put_io_channel(spdk_io_channel_from_ctx(qos->ch));
|
|
spdk_poller_unregister(&qos->poller);
|
|
}
|
|
|
|
free(qos);
|
|
|
|
_spdk_bdev_set_qos_limit_done(ctx, 0);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_disable_qos_msg_done(struct spdk_io_channel_iter *i, int status)
|
|
{
|
|
void *io_device = spdk_io_channel_iter_get_io_device(i);
|
|
struct spdk_bdev *bdev = __bdev_from_io_dev(io_device);
|
|
struct set_qos_limit_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
|
|
struct spdk_thread *thread;
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
thread = bdev->internal.qos->thread;
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
|
|
if (thread != NULL) {
|
|
spdk_thread_send_msg(thread, _spdk_bdev_disable_qos_done, ctx);
|
|
} else {
|
|
_spdk_bdev_disable_qos_done(ctx);
|
|
}
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_disable_qos_msg(struct spdk_io_channel_iter *i)
|
|
{
|
|
struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
|
|
struct spdk_bdev_channel *bdev_ch = spdk_io_channel_get_ctx(ch);
|
|
|
|
bdev_ch->flags &= ~BDEV_CH_QOS_ENABLED;
|
|
|
|
spdk_for_each_channel_continue(i, 0);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_update_qos_rate_limit_msg(void *cb_arg)
|
|
{
|
|
struct set_qos_limit_ctx *ctx = cb_arg;
|
|
struct spdk_bdev *bdev = ctx->bdev;
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
spdk_bdev_qos_update_max_quota_per_timeslice(bdev->internal.qos);
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
|
|
_spdk_bdev_set_qos_limit_done(ctx, 0);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_enable_qos_msg(struct spdk_io_channel_iter *i)
|
|
{
|
|
void *io_device = spdk_io_channel_iter_get_io_device(i);
|
|
struct spdk_bdev *bdev = __bdev_from_io_dev(io_device);
|
|
struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
|
|
struct spdk_bdev_channel *bdev_ch = spdk_io_channel_get_ctx(ch);
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
_spdk_bdev_enable_qos(bdev, bdev_ch);
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
spdk_for_each_channel_continue(i, 0);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_enable_qos_done(struct spdk_io_channel_iter *i, int status)
|
|
{
|
|
struct set_qos_limit_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
|
|
|
|
_spdk_bdev_set_qos_limit_done(ctx, status);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_set_qos_rate_limits(struct spdk_bdev *bdev, uint64_t *limits)
|
|
{
|
|
int i;
|
|
|
|
assert(bdev->internal.qos != NULL);
|
|
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
if (limits[i] != SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {
|
|
bdev->internal.qos->rate_limits[i].limit = limits[i];
|
|
|
|
if (limits[i] == 0) {
|
|
bdev->internal.qos->rate_limits[i].limit =
|
|
SPDK_BDEV_QOS_LIMIT_NOT_DEFINED;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void
|
|
spdk_bdev_set_qos_rate_limits(struct spdk_bdev *bdev, uint64_t *limits,
|
|
void (*cb_fn)(void *cb_arg, int status), void *cb_arg)
|
|
{
|
|
struct set_qos_limit_ctx *ctx;
|
|
uint32_t limit_set_complement;
|
|
uint64_t min_limit_per_sec;
|
|
int i;
|
|
bool disable_rate_limit = true;
|
|
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
if (limits[i] == SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {
|
|
continue;
|
|
}
|
|
|
|
if (limits[i] > 0) {
|
|
disable_rate_limit = false;
|
|
}
|
|
|
|
if (_spdk_bdev_qos_is_iops_rate_limit(i) == true) {
|
|
min_limit_per_sec = SPDK_BDEV_QOS_MIN_IOS_PER_SEC;
|
|
} else {
|
|
/* Change from megabyte to byte rate limit */
|
|
limits[i] = limits[i] * 1024 * 1024;
|
|
min_limit_per_sec = SPDK_BDEV_QOS_MIN_BYTES_PER_SEC;
|
|
}
|
|
|
|
limit_set_complement = limits[i] % min_limit_per_sec;
|
|
if (limit_set_complement) {
|
|
SPDK_ERRLOG("Requested rate limit %" PRIu64 " is not a multiple of %" PRIu64 "\n",
|
|
limits[i], min_limit_per_sec);
|
|
limits[i] += min_limit_per_sec - limit_set_complement;
|
|
SPDK_ERRLOG("Round up the rate limit to %" PRIu64 "\n", limits[i]);
|
|
}
|
|
}
|
|
|
|
ctx = calloc(1, sizeof(*ctx));
|
|
if (ctx == NULL) {
|
|
cb_fn(cb_arg, -ENOMEM);
|
|
return;
|
|
}
|
|
|
|
ctx->cb_fn = cb_fn;
|
|
ctx->cb_arg = cb_arg;
|
|
ctx->bdev = bdev;
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
if (bdev->internal.qos_mod_in_progress) {
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
free(ctx);
|
|
cb_fn(cb_arg, -EAGAIN);
|
|
return;
|
|
}
|
|
bdev->internal.qos_mod_in_progress = true;
|
|
|
|
if (disable_rate_limit == true && bdev->internal.qos) {
|
|
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
|
|
if (limits[i] == SPDK_BDEV_QOS_LIMIT_NOT_DEFINED &&
|
|
(bdev->internal.qos->rate_limits[i].limit > 0 &&
|
|
bdev->internal.qos->rate_limits[i].limit !=
|
|
SPDK_BDEV_QOS_LIMIT_NOT_DEFINED)) {
|
|
disable_rate_limit = false;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (disable_rate_limit == false) {
|
|
if (bdev->internal.qos == NULL) {
|
|
bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos));
|
|
if (!bdev->internal.qos) {
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
SPDK_ERRLOG("Unable to allocate memory for QoS tracking\n");
|
|
free(ctx);
|
|
cb_fn(cb_arg, -ENOMEM);
|
|
return;
|
|
}
|
|
}
|
|
|
|
if (bdev->internal.qos->thread == NULL) {
|
|
/* Enabling */
|
|
_spdk_bdev_set_qos_rate_limits(bdev, limits);
|
|
|
|
spdk_for_each_channel(__bdev_to_io_dev(bdev),
|
|
_spdk_bdev_enable_qos_msg, ctx,
|
|
_spdk_bdev_enable_qos_done);
|
|
} else {
|
|
/* Updating */
|
|
_spdk_bdev_set_qos_rate_limits(bdev, limits);
|
|
|
|
spdk_thread_send_msg(bdev->internal.qos->thread,
|
|
_spdk_bdev_update_qos_rate_limit_msg, ctx);
|
|
}
|
|
} else {
|
|
if (bdev->internal.qos != NULL) {
|
|
_spdk_bdev_set_qos_rate_limits(bdev, limits);
|
|
|
|
/* Disabling */
|
|
spdk_for_each_channel(__bdev_to_io_dev(bdev),
|
|
_spdk_bdev_disable_qos_msg, ctx,
|
|
_spdk_bdev_disable_qos_msg_done);
|
|
} else {
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
_spdk_bdev_set_qos_limit_done(ctx, 0);
|
|
return;
|
|
}
|
|
}
|
|
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
}
|
|
|
|
struct spdk_bdev_histogram_ctx {
|
|
spdk_bdev_histogram_status_cb cb_fn;
|
|
void *cb_arg;
|
|
struct spdk_bdev *bdev;
|
|
int status;
|
|
};
|
|
|
|
static void
|
|
_spdk_bdev_histogram_disable_channel_cb(struct spdk_io_channel_iter *i, int status)
|
|
{
|
|
struct spdk_bdev_histogram_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
|
|
|
|
pthread_mutex_lock(&ctx->bdev->internal.mutex);
|
|
ctx->bdev->internal.histogram_in_progress = false;
|
|
pthread_mutex_unlock(&ctx->bdev->internal.mutex);
|
|
ctx->cb_fn(ctx->cb_arg, ctx->status);
|
|
free(ctx);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_histogram_disable_channel(struct spdk_io_channel_iter *i)
|
|
{
|
|
struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
|
|
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(_ch);
|
|
|
|
if (ch->histogram != NULL) {
|
|
spdk_histogram_data_free(ch->histogram);
|
|
ch->histogram = NULL;
|
|
}
|
|
spdk_for_each_channel_continue(i, 0);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_histogram_enable_channel_cb(struct spdk_io_channel_iter *i, int status)
|
|
{
|
|
struct spdk_bdev_histogram_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
|
|
|
|
if (status != 0) {
|
|
ctx->status = status;
|
|
ctx->bdev->internal.histogram_enabled = false;
|
|
spdk_for_each_channel(__bdev_to_io_dev(ctx->bdev), _spdk_bdev_histogram_disable_channel, ctx,
|
|
_spdk_bdev_histogram_disable_channel_cb);
|
|
} else {
|
|
pthread_mutex_lock(&ctx->bdev->internal.mutex);
|
|
ctx->bdev->internal.histogram_in_progress = false;
|
|
pthread_mutex_unlock(&ctx->bdev->internal.mutex);
|
|
ctx->cb_fn(ctx->cb_arg, ctx->status);
|
|
free(ctx);
|
|
}
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_histogram_enable_channel(struct spdk_io_channel_iter *i)
|
|
{
|
|
struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
|
|
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(_ch);
|
|
int status = 0;
|
|
|
|
if (ch->histogram == NULL) {
|
|
ch->histogram = spdk_histogram_data_alloc();
|
|
if (ch->histogram == NULL) {
|
|
status = -ENOMEM;
|
|
}
|
|
}
|
|
|
|
spdk_for_each_channel_continue(i, status);
|
|
}
|
|
|
|
void
|
|
spdk_bdev_histogram_enable(struct spdk_bdev *bdev, spdk_bdev_histogram_status_cb cb_fn,
|
|
void *cb_arg, bool enable)
|
|
{
|
|
struct spdk_bdev_histogram_ctx *ctx;
|
|
|
|
ctx = calloc(1, sizeof(struct spdk_bdev_histogram_ctx));
|
|
if (ctx == NULL) {
|
|
cb_fn(cb_arg, -ENOMEM);
|
|
return;
|
|
}
|
|
|
|
ctx->bdev = bdev;
|
|
ctx->status = 0;
|
|
ctx->cb_fn = cb_fn;
|
|
ctx->cb_arg = cb_arg;
|
|
|
|
pthread_mutex_lock(&bdev->internal.mutex);
|
|
if (bdev->internal.histogram_in_progress) {
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
free(ctx);
|
|
cb_fn(cb_arg, -EAGAIN);
|
|
return;
|
|
}
|
|
|
|
bdev->internal.histogram_in_progress = true;
|
|
pthread_mutex_unlock(&bdev->internal.mutex);
|
|
|
|
bdev->internal.histogram_enabled = enable;
|
|
|
|
if (enable) {
|
|
/* Allocate histogram for each channel */
|
|
spdk_for_each_channel(__bdev_to_io_dev(bdev), _spdk_bdev_histogram_enable_channel, ctx,
|
|
_spdk_bdev_histogram_enable_channel_cb);
|
|
} else {
|
|
spdk_for_each_channel(__bdev_to_io_dev(bdev), _spdk_bdev_histogram_disable_channel, ctx,
|
|
_spdk_bdev_histogram_disable_channel_cb);
|
|
}
|
|
}
|
|
|
|
struct spdk_bdev_histogram_data_ctx {
|
|
spdk_bdev_histogram_data_cb cb_fn;
|
|
void *cb_arg;
|
|
struct spdk_bdev *bdev;
|
|
/** merged histogram data from all channels */
|
|
struct spdk_histogram_data *histogram;
|
|
};
|
|
|
|
static void
|
|
_spdk_bdev_histogram_get_channel_cb(struct spdk_io_channel_iter *i, int status)
|
|
{
|
|
struct spdk_bdev_histogram_data_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
|
|
|
|
ctx->cb_fn(ctx->cb_arg, status, ctx->histogram);
|
|
free(ctx);
|
|
}
|
|
|
|
static void
|
|
_spdk_bdev_histogram_get_channel(struct spdk_io_channel_iter *i)
|
|
{
|
|
struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
|
|
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(_ch);
|
|
struct spdk_bdev_histogram_data_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
|
|
int status = 0;
|
|
|
|
if (ch->histogram == NULL) {
|
|
status = -EFAULT;
|
|
} else {
|
|
spdk_histogram_data_merge(ctx->histogram, ch->histogram);
|
|
}
|
|
|
|
spdk_for_each_channel_continue(i, status);
|
|
}
|
|
|
|
void
|
|
spdk_bdev_histogram_get(struct spdk_bdev *bdev, struct spdk_histogram_data *histogram,
|
|
spdk_bdev_histogram_data_cb cb_fn,
|
|
void *cb_arg)
|
|
{
|
|
struct spdk_bdev_histogram_data_ctx *ctx;
|
|
|
|
ctx = calloc(1, sizeof(struct spdk_bdev_histogram_data_ctx));
|
|
if (ctx == NULL) {
|
|
cb_fn(cb_arg, -ENOMEM, NULL);
|
|
return;
|
|
}
|
|
|
|
ctx->bdev = bdev;
|
|
ctx->cb_fn = cb_fn;
|
|
ctx->cb_arg = cb_arg;
|
|
|
|
ctx->histogram = histogram;
|
|
|
|
spdk_for_each_channel(__bdev_to_io_dev(bdev), _spdk_bdev_histogram_get_channel, ctx,
|
|
_spdk_bdev_histogram_get_channel_cb);
|
|
}
|
|
|
|
SPDK_LOG_REGISTER_COMPONENT("bdev", SPDK_LOG_BDEV)
|
|
|
|
SPDK_TRACE_REGISTER_FN(bdev_trace, "bdev", TRACE_GROUP_BDEV)
|
|
{
|
|
spdk_trace_register_owner(OWNER_BDEV, 'b');
|
|
spdk_trace_register_object(OBJECT_BDEV_IO, 'i');
|
|
spdk_trace_register_description("BDEV_IO_START", "", TRACE_BDEV_IO_START, OWNER_BDEV,
|
|
OBJECT_BDEV_IO, 1, 0, "type: ");
|
|
spdk_trace_register_description("BDEV_IO_DONE", "", TRACE_BDEV_IO_DONE, OWNER_BDEV,
|
|
OBJECT_BDEV_IO, 0, 0, "");
|
|
}
|