d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
076d14ce4b
numam-spdk/lib/bdev/bdev.c
Shuhei Matsumoto cf8405fc24 bdev: Hold mutex while removing name from name tree 
We had not held mutex while removing bdev name or alias from bdev
name tree for most cases. Fix these in this patch.

spdk_bdev_unregister() already holds g_bdev_mgr.mutex when removing
name, and so we do not need to change it.

spdk_bdev_close() had not held g_bdev_mgr.mutex. What we want to lock
is only when removing name from name tree, that is, calling
bdev_name_del() in bdev_unregister_unsafe(). However, we need to
keep hierarchical lock ordering. Hence get and free g_bdev_mgr.mutex
outside of bdev->internal.mutex.

Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: I4e2c8604e27c8603725efa9bc0bee2013eccb2ac
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/8527
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
2021-07-12 15:30:39 +00:00

7023 lines
190 KiB
C
Raw Blame History

/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation. All rights reserved.
* Copyright (c) 2019 Mellanox Technologies LTD. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "spdk/stdinc.h"
#include "spdk/bdev.h"
#include "spdk/config.h"
#include "spdk/env.h"
#include "spdk/thread.h"
#include "spdk/likely.h"
#include "spdk/queue.h"
#include "spdk/nvme_spec.h"
#include "spdk/scsi_spec.h"
#include "spdk/notify.h"
#include "spdk/util.h"
#include "spdk/trace.h"
#include "spdk/bdev_module.h"
#include "spdk/log.h"
#include "spdk/string.h"
#include "bdev_internal.h"
#ifdef SPDK_CONFIG_VTUNE
#include "ittnotify.h"
#include "ittnotify_types.h"
int __itt_init_ittlib(const char *, __itt_group_id);
#endif
#define SPDK_BDEV_IO_POOL_SIZE (64 * 1024 - 1)
#define SPDK_BDEV_IO_CACHE_SIZE 256
#define SPDK_BDEV_AUTO_EXAMINE true
#define BUF_SMALL_POOL_SIZE 8191
#define BUF_LARGE_POOL_SIZE 1023
#define NOMEM_THRESHOLD_COUNT 8
#define ZERO_BUFFER_SIZE 0x100000
#define OWNER_BDEV 0x2
#define OBJECT_BDEV_IO 0x2
#define TRACE_GROUP_BDEV 0x3
#define TRACE_BDEV_IO_START SPDK_TPOINT_ID(TRACE_GROUP_BDEV, 0x0)
#define TRACE_BDEV_IO_DONE SPDK_TPOINT_ID(TRACE_GROUP_BDEV, 0x1)
#define SPDK_BDEV_QOS_TIMESLICE_IN_USEC 1000
#define SPDK_BDEV_QOS_MIN_IO_PER_TIMESLICE 1
#define SPDK_BDEV_QOS_MIN_BYTE_PER_TIMESLICE 512
#define SPDK_BDEV_QOS_MIN_IOS_PER_SEC 1000
#define SPDK_BDEV_QOS_MIN_BYTES_PER_SEC (1024 * 1024)
#define SPDK_BDEV_QOS_LIMIT_NOT_DEFINED UINT64_MAX
#define SPDK_BDEV_IO_POLL_INTERVAL_IN_MSEC 1000
#define SPDK_BDEV_POOL_ALIGNMENT 512
/* The maximum number of children requests for a UNMAP or WRITE ZEROES command
* when splitting into children requests at a time.
*/
#define SPDK_BDEV_MAX_CHILDREN_UNMAP_WRITE_ZEROES_REQS (8)
static const char *qos_rpc_type[] = {"rw_ios_per_sec",
"rw_mbytes_per_sec", "r_mbytes_per_sec", "w_mbytes_per_sec"
};
TAILQ_HEAD(spdk_bdev_list, spdk_bdev);
RB_HEAD(bdev_name_tree, spdk_bdev_name);
static int
bdev_name_cmp(struct spdk_bdev_name *name1, struct spdk_bdev_name *name2)
{
return strcmp(name1->name, name2->name);
}
RB_GENERATE_STATIC(bdev_name_tree, spdk_bdev_name, node, bdev_name_cmp);
struct spdk_bdev_mgr {
struct spdk_mempool *bdev_io_pool;
struct spdk_mempool *buf_small_pool;
struct spdk_mempool *buf_large_pool;
void *zero_buffer;
TAILQ_HEAD(bdev_module_list, spdk_bdev_module) bdev_modules;
struct spdk_bdev_list bdevs;
struct bdev_name_tree bdev_names;
bool init_complete;
bool module_init_complete;
pthread_mutex_t mutex;
#ifdef SPDK_CONFIG_VTUNE
__itt_domain *domain;
#endif
};
static struct spdk_bdev_mgr g_bdev_mgr = {
.bdev_modules = TAILQ_HEAD_INITIALIZER(g_bdev_mgr.bdev_modules),
.bdevs = TAILQ_HEAD_INITIALIZER(g_bdev_mgr.bdevs),
.bdev_names = RB_INITIALIZER(g_bdev_mgr.bdev_names),
.init_complete = false,
.module_init_complete = false,
.mutex = PTHREAD_MUTEX_INITIALIZER,
};
typedef void (*lock_range_cb)(void *ctx, int status);
struct lba_range {
uint64_t offset;
uint64_t length;
void *locked_ctx;
struct spdk_bdev_channel *owner_ch;
TAILQ_ENTRY(lba_range) tailq;
};
static struct spdk_bdev_opts g_bdev_opts = {
.bdev_io_pool_size = SPDK_BDEV_IO_POOL_SIZE,
.bdev_io_cache_size = SPDK_BDEV_IO_CACHE_SIZE,
.bdev_auto_examine = SPDK_BDEV_AUTO_EXAMINE,
.small_buf_pool_size = BUF_SMALL_POOL_SIZE,
.large_buf_pool_size = BUF_LARGE_POOL_SIZE,
};
static spdk_bdev_init_cb g_init_cb_fn = NULL;
static void *g_init_cb_arg = NULL;
static spdk_bdev_fini_cb g_fini_cb_fn = NULL;
static void *g_fini_cb_arg = NULL;
static struct spdk_thread *g_fini_thread = NULL;
struct spdk_bdev_qos_limit {
/** IOs or bytes allowed per second (i.e., 1s). */
uint64_t limit;
/** Remaining IOs or bytes allowed in current timeslice (e.g., 1ms).
* For remaining bytes, allowed to run negative if an I/O is submitted when
* some bytes are remaining, but the I/O is bigger than that amount. The
* excess will be deducted from the next timeslice.
*/
int64_t remaining_this_timeslice;
/** Minimum allowed IOs or bytes to be issued in one timeslice (e.g., 1ms). */
uint32_t min_per_timeslice;
/** Maximum allowed IOs or bytes to be issued in one timeslice (e.g., 1ms). */
uint32_t max_per_timeslice;
/** Function to check whether to queue the IO. */
bool (*queue_io)(const struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io);
/** Function to update for the submitted IO. */
void (*update_quota)(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io);
};
struct spdk_bdev_qos {
/** Types of structure of rate limits. */
struct spdk_bdev_qos_limit rate_limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES];
/** The channel that all I/O are funneled through. */
struct spdk_bdev_channel *ch;
/** The thread on which the poller is running. */
struct spdk_thread *thread;
/** Queue of I/O waiting to be issued. */
bdev_io_tailq_t queued;
/** Size of a timeslice in tsc ticks. */
uint64_t timeslice_size;
/** Timestamp of start of last timeslice. */
uint64_t last_timeslice;
/** Poller that processes queued I/O commands each time slice. */
struct spdk_poller *poller;
};
struct spdk_bdev_mgmt_channel {
bdev_io_stailq_t need_buf_small;
bdev_io_stailq_t need_buf_large;
/*
* Each thread keeps a cache of bdev_io - this allows
* bdev threads which are *not* DPDK threads to still
* benefit from a per-thread bdev_io cache. Without
* this, non-DPDK threads fetching from the mempool
* incur a cmpxchg on get and put.
*/
bdev_io_stailq_t per_thread_cache;
uint32_t per_thread_cache_count;
uint32_t bdev_io_cache_size;
TAILQ_HEAD(, spdk_bdev_shared_resource) shared_resources;
TAILQ_HEAD(, spdk_bdev_io_wait_entry) io_wait_queue;
};
/*
* Per-module (or per-io_device) data. Multiple bdevs built on the same io_device
* will queue here their IO that awaits retry. It makes it possible to retry sending
* IO to one bdev after IO from other bdev completes.
*/
struct spdk_bdev_shared_resource {
/* The bdev management channel */
struct spdk_bdev_mgmt_channel *mgmt_ch;
/*
* Count of I/O submitted to bdev module and waiting for completion.
* Incremented before submit_request() is called on an spdk_bdev_io.
*/
uint64_t io_outstanding;
/*
* Queue of IO awaiting retry because of a previous NOMEM status returned
* on this channel.
*/
bdev_io_tailq_t nomem_io;
/*
* Threshold which io_outstanding must drop to before retrying nomem_io.
*/
uint64_t nomem_threshold;
/* I/O channel allocated by a bdev module */
struct spdk_io_channel *shared_ch;
/* Refcount of bdev channels using this resource */
uint32_t ref;
TAILQ_ENTRY(spdk_bdev_shared_resource) link;
};
#define BDEV_CH_RESET_IN_PROGRESS (1 << 0)
#define BDEV_CH_QOS_ENABLED (1 << 1)
struct spdk_bdev_channel {
struct spdk_bdev *bdev;
/* The channel for the underlying device */
struct spdk_io_channel *channel;
/* Per io_device per thread data */
struct spdk_bdev_shared_resource *shared_resource;
struct spdk_bdev_io_stat stat;
/*
* Count of I/O submitted to the underlying dev module through this channel
* and waiting for completion.
*/
uint64_t io_outstanding;
/*
* List of all submitted I/Os including I/O that are generated via splitting.
*/
bdev_io_tailq_t io_submitted;
/*
* List of spdk_bdev_io that are currently queued because they write to a locked
* LBA range.
*/
bdev_io_tailq_t io_locked;
uint32_t flags;
struct spdk_histogram_data *histogram;
#ifdef SPDK_CONFIG_VTUNE
uint64_t start_tsc;
uint64_t interval_tsc;
__itt_string_handle *handle;
struct spdk_bdev_io_stat prev_stat;
#endif
bdev_io_tailq_t queued_resets;
lba_range_tailq_t locked_ranges;
};
struct media_event_entry {
struct spdk_bdev_media_event event;
TAILQ_ENTRY(media_event_entry) tailq;
};
#define MEDIA_EVENT_POOL_SIZE 64
struct spdk_bdev_desc {
struct spdk_bdev *bdev;
struct spdk_thread *thread;
struct {
spdk_bdev_event_cb_t event_fn;
void *ctx;
} callback;
bool closed;
bool write;
pthread_mutex_t mutex;
uint32_t refs;
TAILQ_HEAD(, media_event_entry) pending_media_events;
TAILQ_HEAD(, media_event_entry) free_media_events;
struct media_event_entry *media_events_buffer;
TAILQ_ENTRY(spdk_bdev_desc) link;
uint64_t timeout_in_sec;
spdk_bdev_io_timeout_cb cb_fn;
void *cb_arg;
struct spdk_poller *io_timeout_poller;
};
struct spdk_bdev_iostat_ctx {
struct spdk_bdev_io_stat *stat;
spdk_bdev_get_device_stat_cb cb;
void *cb_arg;
};
struct set_qos_limit_ctx {
void (*cb_fn)(void *cb_arg, int status);
void *cb_arg;
struct spdk_bdev *bdev;
};
#define __bdev_to_io_dev(bdev) (((char *)bdev) + 1)
#define __bdev_from_io_dev(io_dev) ((struct spdk_bdev *)(((char *)io_dev) - 1))
static void bdev_write_zero_buffer_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg);
static void bdev_write_zero_buffer_next(void *_bdev_io);
static void bdev_enable_qos_msg(struct spdk_io_channel_iter *i);
static void bdev_enable_qos_done(struct spdk_io_channel_iter *i, int status);
static int
bdev_readv_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt, void *md_buf, uint64_t offset_blocks,
uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg);
static int
bdev_writev_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt, void *md_buf,
uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg);
static int
bdev_lock_lba_range(struct spdk_bdev_desc *desc, struct spdk_io_channel *_ch,
uint64_t offset, uint64_t length,
lock_range_cb cb_fn, void *cb_arg);
static int
bdev_unlock_lba_range(struct spdk_bdev_desc *desc, struct spdk_io_channel *_ch,
uint64_t offset, uint64_t length,
lock_range_cb cb_fn, void *cb_arg);
static inline void bdev_io_complete(void *ctx);
static bool bdev_abort_queued_io(bdev_io_tailq_t *queue, struct spdk_bdev_io *bio_to_abort);
static bool bdev_abort_buf_io(bdev_io_stailq_t *queue, struct spdk_bdev_io *bio_to_abort);
void
spdk_bdev_get_opts(struct spdk_bdev_opts *opts, size_t opts_size)
{
if (!opts) {
SPDK_ERRLOG("opts should not be NULL\n");
return;
}
if (!opts_size) {
SPDK_ERRLOG("opts_size should not be zero value\n");
return;
}
opts->opts_size = opts_size;
#define SET_FIELD(field) \
if (offsetof(struct spdk_bdev_opts, field) + sizeof(opts->field) <= opts_size) { \
opts->field = g_bdev_opts.field; \
} \
SET_FIELD(bdev_io_pool_size);
SET_FIELD(bdev_io_cache_size);
SET_FIELD(bdev_auto_examine);
SET_FIELD(small_buf_pool_size);
SET_FIELD(large_buf_pool_size);
/* Do not remove this statement, you should always update this statement when you adding a new field,
* and do not forget to add the SET_FIELD statement for your added field. */
SPDK_STATIC_ASSERT(sizeof(struct spdk_bdev_opts) == 32, "Incorrect size");
#undef SET_FIELD
}
int
spdk_bdev_set_opts(struct spdk_bdev_opts *opts)
{
uint32_t min_pool_size;
if (!opts) {
SPDK_ERRLOG("opts cannot be NULL\n");
return -1;
}
if (!opts->opts_size) {
SPDK_ERRLOG("opts_size inside opts cannot be zero value\n");
return -1;
}
/*
* Add 1 to the thread count to account for the extra mgmt_ch that gets created during subsystem
* initialization. A second mgmt_ch will be created on the same thread when the application starts
* but before the deferred put_io_channel event is executed for the first mgmt_ch.
*/
min_pool_size = opts->bdev_io_cache_size * (spdk_thread_get_count() + 1);
if (opts->bdev_io_pool_size < min_pool_size) {
SPDK_ERRLOG("bdev_io_pool_size %" PRIu32 " is not compatible with bdev_io_cache_size %" PRIu32
" and %" PRIu32 " threads\n", opts->bdev_io_pool_size, opts->bdev_io_cache_size,
spdk_thread_get_count());
SPDK_ERRLOG("bdev_io_pool_size must be at least %" PRIu32 "\n", min_pool_size);
return -1;
}
if (opts->small_buf_pool_size < BUF_SMALL_POOL_SIZE) {
SPDK_ERRLOG("small_buf_pool_size must be at least %" PRIu32 "\n", BUF_SMALL_POOL_SIZE);
return -1;
}
if (opts->large_buf_pool_size < BUF_LARGE_POOL_SIZE) {
SPDK_ERRLOG("large_buf_pool_size must be at least %" PRIu32 "\n", BUF_LARGE_POOL_SIZE);
return -1;
}
#define SET_FIELD(field) \
if (offsetof(struct spdk_bdev_opts, field) + sizeof(opts->field) <= opts->opts_size) { \
g_bdev_opts.field = opts->field; \
} \
SET_FIELD(bdev_io_pool_size);
SET_FIELD(bdev_io_cache_size);
SET_FIELD(bdev_auto_examine);
SET_FIELD(small_buf_pool_size);
SET_FIELD(large_buf_pool_size);
g_bdev_opts.opts_size = opts->opts_size;
#undef SET_FIELD
return 0;
}
static struct spdk_bdev *
bdev_get_by_name(const char *bdev_name)
{
struct spdk_bdev_name find;
struct spdk_bdev_name *res;
find.name = (char *)bdev_name;
res = RB_FIND(bdev_name_tree, &g_bdev_mgr.bdev_names, &find);
if (res != NULL) {
return res->bdev;
}
return NULL;
}
struct spdk_bdev *
spdk_bdev_get_by_name(const char *bdev_name)
{
struct spdk_bdev *bdev;
pthread_mutex_lock(&g_bdev_mgr.mutex);
bdev = bdev_get_by_name(bdev_name);
pthread_mutex_unlock(&g_bdev_mgr.mutex);
return bdev;
}
struct spdk_bdev_wait_for_examine_ctx {
struct spdk_poller *poller;
spdk_bdev_wait_for_examine_cb cb_fn;
void *cb_arg;
};
static bool
bdev_module_all_actions_completed(void);
static int
bdev_wait_for_examine_cb(void *arg)
{
struct spdk_bdev_wait_for_examine_ctx *ctx = arg;
if (!bdev_module_all_actions_completed()) {
return SPDK_POLLER_IDLE;
}
spdk_poller_unregister(&ctx->poller);
ctx->cb_fn(ctx->cb_arg);
free(ctx);
return SPDK_POLLER_BUSY;
}
int
spdk_bdev_wait_for_examine(spdk_bdev_wait_for_examine_cb cb_fn, void *cb_arg)
{
struct spdk_bdev_wait_for_examine_ctx *ctx;
ctx = calloc(1, sizeof(*ctx));
if (ctx == NULL) {
return -ENOMEM;
}
ctx->cb_fn = cb_fn;
ctx->cb_arg = cb_arg;
ctx->poller = SPDK_POLLER_REGISTER(bdev_wait_for_examine_cb, ctx, 0);
return 0;
}
struct spdk_bdev_examine_item {
char *name;
TAILQ_ENTRY(spdk_bdev_examine_item) link;
};
TAILQ_HEAD(spdk_bdev_examine_allowlist, spdk_bdev_examine_item);
struct spdk_bdev_examine_allowlist g_bdev_examine_allowlist = TAILQ_HEAD_INITIALIZER(
g_bdev_examine_allowlist);
static inline bool
bdev_examine_allowlist_check(const char *name)
{
struct spdk_bdev_examine_item *item;
TAILQ_FOREACH(item, &g_bdev_examine_allowlist, link) {
if (strcmp(name, item->name) == 0) {
return true;
}
}
return false;
}
static inline void
bdev_examine_allowlist_free(void)
{
struct spdk_bdev_examine_item *item;
while (!TAILQ_EMPTY(&g_bdev_examine_allowlist)) {
item = TAILQ_FIRST(&g_bdev_examine_allowlist);
TAILQ_REMOVE(&g_bdev_examine_allowlist, item, link);
free(item->name);
free(item);
}
}
static inline bool
bdev_in_examine_allowlist(struct spdk_bdev *bdev)
{
struct spdk_bdev_alias *tmp;
if (bdev_examine_allowlist_check(bdev->name)) {
return true;
}
TAILQ_FOREACH(tmp, &bdev->aliases, tailq) {
if (bdev_examine_allowlist_check(tmp->alias.name)) {
return true;
}
}
return false;
}
static inline bool
bdev_ok_to_examine(struct spdk_bdev *bdev)
{
if (g_bdev_opts.bdev_auto_examine) {
return true;
} else {
return bdev_in_examine_allowlist(bdev);
}
}
static void
bdev_examine(struct spdk_bdev *bdev)
{
struct spdk_bdev_module *module;
uint32_t action;
TAILQ_FOREACH(module, &g_bdev_mgr.bdev_modules, internal.tailq) {
if (module->examine_config && bdev_ok_to_examine(bdev)) {
action = module->internal.action_in_progress;
module->internal.action_in_progress++;
module->examine_config(bdev);
if (action != module->internal.action_in_progress) {
SPDK_ERRLOG("examine_config for module %s did not call spdk_bdev_module_examine_done()\n",
module->name);
}
}
}
if (bdev->internal.claim_module && bdev_ok_to_examine(bdev)) {
if (bdev->internal.claim_module->examine_disk) {
bdev->internal.claim_module->internal.action_in_progress++;
bdev->internal.claim_module->examine_disk(bdev);
}
return;
}
TAILQ_FOREACH(module, &g_bdev_mgr.bdev_modules, internal.tailq) {
if (module->examine_disk && bdev_ok_to_examine(bdev)) {
module->internal.action_in_progress++;
module->examine_disk(bdev);
}
}
}
int
spdk_bdev_examine(const char *name)
{
struct spdk_bdev *bdev;
struct spdk_bdev_examine_item *item;
if (g_bdev_opts.bdev_auto_examine) {
SPDK_ERRLOG("Manual examine is not allowed if auto examine is enabled");
return -EINVAL;
}
if (bdev_examine_allowlist_check(name)) {
SPDK_ERRLOG("Duplicate bdev name for manual examine: %s\n", name);
return -EEXIST;
}
item = calloc(1, sizeof(*item));
if (!item) {
return -ENOMEM;
}
item->name = strdup(name);
if (!item->name) {
free(item);
return -ENOMEM;
}
TAILQ_INSERT_TAIL(&g_bdev_examine_allowlist, item, link);
bdev = spdk_bdev_get_by_name(name);
if (bdev) {
bdev_examine(bdev);
}
return 0;
}
static inline void
bdev_examine_allowlist_config_json(struct spdk_json_write_ctx *w)
{
struct spdk_bdev_examine_item *item;
TAILQ_FOREACH(item, &g_bdev_examine_allowlist, link) {
spdk_json_write_object_begin(w);
spdk_json_write_named_string(w, "method", "bdev_examine");
spdk_json_write_named_object_begin(w, "params");
spdk_json_write_named_string(w, "name", item->name);
spdk_json_write_object_end(w);
spdk_json_write_object_end(w);
}
}
struct spdk_bdev *
spdk_bdev_first(void)
{
struct spdk_bdev *bdev;
bdev = TAILQ_FIRST(&g_bdev_mgr.bdevs);
if (bdev) {
SPDK_DEBUGLOG(bdev, "Starting bdev iteration at %s\n", bdev->name);
}
return bdev;
}
struct spdk_bdev *
spdk_bdev_next(struct spdk_bdev *prev)
{
struct spdk_bdev *bdev;
bdev = TAILQ_NEXT(prev, internal.link);
if (bdev) {
SPDK_DEBUGLOG(bdev, "Continuing bdev iteration at %s\n", bdev->name);
}
return bdev;
}
static struct spdk_bdev *
_bdev_next_leaf(struct spdk_bdev *bdev)
{
while (bdev != NULL) {
if (bdev->internal.claim_module == NULL) {
return bdev;
} else {
bdev = TAILQ_NEXT(bdev, internal.link);
}
}
return bdev;
}
struct spdk_bdev *
spdk_bdev_first_leaf(void)
{
struct spdk_bdev *bdev;
bdev = _bdev_next_leaf(TAILQ_FIRST(&g_bdev_mgr.bdevs));
if (bdev) {
SPDK_DEBUGLOG(bdev, "Starting bdev iteration at %s\n", bdev->name);
}
return bdev;
}
struct spdk_bdev *
spdk_bdev_next_leaf(struct spdk_bdev *prev)
{
struct spdk_bdev *bdev;
bdev = _bdev_next_leaf(TAILQ_NEXT(prev, internal.link));
if (bdev) {
SPDK_DEBUGLOG(bdev, "Continuing bdev iteration at %s\n", bdev->name);
}
return bdev;
}
void
spdk_bdev_io_set_buf(struct spdk_bdev_io *bdev_io, void *buf, size_t len)
{
struct iovec *iovs;
if (bdev_io->u.bdev.iovs == NULL) {
bdev_io->u.bdev.iovs = &bdev_io->iov;
bdev_io->u.bdev.iovcnt = 1;
}
iovs = bdev_io->u.bdev.iovs;
assert(iovs != NULL);
assert(bdev_io->u.bdev.iovcnt >= 1);
iovs[0].iov_base = buf;
iovs[0].iov_len = len;
}
void
spdk_bdev_io_set_md_buf(struct spdk_bdev_io *bdev_io, void *md_buf, size_t len)
{
assert((len / spdk_bdev_get_md_size(bdev_io->bdev)) >= bdev_io->u.bdev.num_blocks);
bdev_io->u.bdev.md_buf = md_buf;
}
static bool
_is_buf_allocated(const struct iovec *iovs)
{
if (iovs == NULL) {
return false;
}
return iovs[0].iov_base != NULL;
}
static bool
_are_iovs_aligned(struct iovec *iovs, int iovcnt, uint32_t alignment)
{
int i;
uintptr_t iov_base;
if (spdk_likely(alignment == 1)) {
return true;
}
for (i = 0; i < iovcnt; i++) {
iov_base = (uintptr_t)iovs[i].iov_base;
if ((iov_base & (alignment - 1)) != 0) {
return false;
}
}
return true;
}
static void
_copy_iovs_to_buf(void *buf, size_t buf_len, struct iovec *iovs, int iovcnt)
{
int i;
size_t len;
for (i = 0; i < iovcnt; i++) {
len = spdk_min(iovs[i].iov_len, buf_len);
memcpy(buf, iovs[i].iov_base, len);
buf += len;
buf_len -= len;
}
}
static void
_copy_buf_to_iovs(struct iovec *iovs, int iovcnt, void *buf, size_t buf_len)
{
int i;
size_t len;
for (i = 0; i < iovcnt; i++) {
len = spdk_min(iovs[i].iov_len, buf_len);
memcpy(iovs[i].iov_base, buf, len);
buf += len;
buf_len -= len;
}
}
static void
_bdev_io_set_bounce_buf(struct spdk_bdev_io *bdev_io, void *buf, size_t len)
{
/* save original iovec */
bdev_io->internal.orig_iovs = bdev_io->u.bdev.iovs;
bdev_io->internal.orig_iovcnt = bdev_io->u.bdev.iovcnt;
/* set bounce iov */
bdev_io->u.bdev.iovs = &bdev_io->internal.bounce_iov;
bdev_io->u.bdev.iovcnt = 1;
/* set bounce buffer for this operation */
bdev_io->u.bdev.iovs[0].iov_base = buf;
bdev_io->u.bdev.iovs[0].iov_len = len;
/* if this is write path, copy data from original buffer to bounce buffer */
if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) {
_copy_iovs_to_buf(buf, len, bdev_io->internal.orig_iovs, bdev_io->internal.orig_iovcnt);
}
}
static void
_bdev_io_set_bounce_md_buf(struct spdk_bdev_io *bdev_io, void *md_buf, size_t len)
{
/* save original md_buf */
bdev_io->internal.orig_md_buf = bdev_io->u.bdev.md_buf;
/* set bounce md_buf */
bdev_io->u.bdev.md_buf = md_buf;
if (bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) {
memcpy(md_buf, bdev_io->internal.orig_md_buf, len);
}
}
static void
bdev_io_get_buf_complete(struct spdk_bdev_io *bdev_io, void *buf, bool status)
{
struct spdk_io_channel *ch = spdk_bdev_io_get_io_channel(bdev_io);
if (spdk_unlikely(bdev_io->internal.get_aux_buf_cb != NULL)) {
bdev_io->internal.get_aux_buf_cb(ch, bdev_io, buf);
bdev_io->internal.get_aux_buf_cb = NULL;
} else {
assert(bdev_io->internal.get_buf_cb != NULL);
bdev_io->internal.buf = buf;
bdev_io->internal.get_buf_cb(ch, bdev_io, status);
bdev_io->internal.get_buf_cb = NULL;
}
}
static void
_bdev_io_set_buf(struct spdk_bdev_io *bdev_io, void *buf, uint64_t len)
{
struct spdk_bdev *bdev = bdev_io->bdev;
bool buf_allocated;
uint64_t md_len, alignment;
void *aligned_buf;
if (spdk_unlikely(bdev_io->internal.get_aux_buf_cb != NULL)) {
bdev_io_get_buf_complete(bdev_io, buf, true);
return;
}
alignment = spdk_bdev_get_buf_align(bdev);
buf_allocated = _is_buf_allocated(bdev_io->u.bdev.iovs);
aligned_buf = (void *)(((uintptr_t)buf + (alignment - 1)) & ~(alignment - 1));
if (buf_allocated) {
_bdev_io_set_bounce_buf(bdev_io, aligned_buf, len);
} else {
spdk_bdev_io_set_buf(bdev_io, aligned_buf, len);
}
if (spdk_bdev_is_md_separate(bdev)) {
aligned_buf = (char *)aligned_buf + len;
md_len = bdev_io->u.bdev.num_blocks * bdev->md_len;
assert(((uintptr_t)aligned_buf & (alignment - 1)) == 0);
if (bdev_io->u.bdev.md_buf != NULL) {
_bdev_io_set_bounce_md_buf(bdev_io, aligned_buf, md_len);
} else {
spdk_bdev_io_set_md_buf(bdev_io, aligned_buf, md_len);
}
}
bdev_io_get_buf_complete(bdev_io, buf, true);
}
static void
_bdev_io_put_buf(struct spdk_bdev_io *bdev_io, void *buf, uint64_t buf_len)
{
struct spdk_bdev *bdev = bdev_io->bdev;
struct spdk_mempool *pool;
struct spdk_bdev_io *tmp;
bdev_io_stailq_t *stailq;
struct spdk_bdev_mgmt_channel *ch;
uint64_t md_len, alignment;
md_len = spdk_bdev_is_md_separate(bdev) ? bdev_io->u.bdev.num_blocks * bdev->md_len : 0;
alignment = spdk_bdev_get_buf_align(bdev);
ch = bdev_io->internal.ch->shared_resource->mgmt_ch;
if (buf_len + alignment + md_len <= SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_SMALL_BUF_MAX_SIZE) +
SPDK_BDEV_POOL_ALIGNMENT) {
pool = g_bdev_mgr.buf_small_pool;
stailq = &ch->need_buf_small;
} else {
pool = g_bdev_mgr.buf_large_pool;
stailq = &ch->need_buf_large;
}
if (STAILQ_EMPTY(stailq)) {
spdk_mempool_put(pool, buf);
} else {
tmp = STAILQ_FIRST(stailq);
STAILQ_REMOVE_HEAD(stailq, internal.buf_link);
_bdev_io_set_buf(tmp, buf, tmp->internal.buf_len);
}
}
static void
bdev_io_put_buf(struct spdk_bdev_io *bdev_io)
{
assert(bdev_io->internal.buf != NULL);
_bdev_io_put_buf(bdev_io, bdev_io->internal.buf, bdev_io->internal.buf_len);
bdev_io->internal.buf = NULL;
}
void
spdk_bdev_io_put_aux_buf(struct spdk_bdev_io *bdev_io, void *buf)
{
uint64_t len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
assert(buf != NULL);
_bdev_io_put_buf(bdev_io, buf, len);
}
static void
_bdev_io_unset_bounce_buf(struct spdk_bdev_io *bdev_io)
{
if (spdk_likely(bdev_io->internal.orig_iovcnt == 0)) {
assert(bdev_io->internal.orig_md_buf == NULL);
return;
}
/* if this is read path, copy data from bounce buffer to original buffer */
if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ &&
bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS) {
_copy_buf_to_iovs(bdev_io->internal.orig_iovs,
bdev_io->internal.orig_iovcnt,
bdev_io->internal.bounce_iov.iov_base,
bdev_io->internal.bounce_iov.iov_len);
}
/* set original buffer for this io */
bdev_io->u.bdev.iovcnt = bdev_io->internal.orig_iovcnt;
bdev_io->u.bdev.iovs = bdev_io->internal.orig_iovs;
/* disable bouncing buffer for this io */
bdev_io->internal.orig_iovcnt = 0;
bdev_io->internal.orig_iovs = NULL;
/* do the same for metadata buffer */
if (spdk_unlikely(bdev_io->internal.orig_md_buf != NULL)) {
assert(spdk_bdev_is_md_separate(bdev_io->bdev));
if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ &&
bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS) {
memcpy(bdev_io->internal.orig_md_buf, bdev_io->u.bdev.md_buf,
bdev_io->u.bdev.num_blocks * spdk_bdev_get_md_size(bdev_io->bdev));
}
bdev_io->u.bdev.md_buf = bdev_io->internal.orig_md_buf;
bdev_io->internal.orig_md_buf = NULL;
}
/* We want to free the bounce buffer here since we know we're done with it (as opposed
* to waiting for the conditional free of internal.buf in spdk_bdev_free_io()).
*/
bdev_io_put_buf(bdev_io);
}
static void
bdev_io_get_buf(struct spdk_bdev_io *bdev_io, uint64_t len)
{
struct spdk_bdev *bdev = bdev_io->bdev;
struct spdk_mempool *pool;
bdev_io_stailq_t *stailq;
struct spdk_bdev_mgmt_channel *mgmt_ch;
uint64_t alignment, md_len;
void *buf;
alignment = spdk_bdev_get_buf_align(bdev);
md_len = spdk_bdev_is_md_separate(bdev) ? bdev_io->u.bdev.num_blocks * bdev->md_len : 0;
if (len + alignment + md_len > SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_LARGE_BUF_MAX_SIZE) +
SPDK_BDEV_POOL_ALIGNMENT) {
SPDK_ERRLOG("Length + alignment %" PRIu64 " is larger than allowed\n",
len + alignment);
bdev_io_get_buf_complete(bdev_io, NULL, false);
return;
}
mgmt_ch = bdev_io->internal.ch->shared_resource->mgmt_ch;
bdev_io->internal.buf_len = len;
if (len + alignment + md_len <= SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_SMALL_BUF_MAX_SIZE) +
SPDK_BDEV_POOL_ALIGNMENT) {
pool = g_bdev_mgr.buf_small_pool;
stailq = &mgmt_ch->need_buf_small;
} else {
pool = g_bdev_mgr.buf_large_pool;
stailq = &mgmt_ch->need_buf_large;
}
buf = spdk_mempool_get(pool);
if (!buf) {
STAILQ_INSERT_TAIL(stailq, bdev_io, internal.buf_link);
} else {
_bdev_io_set_buf(bdev_io, buf, len);
}
}
void
spdk_bdev_io_get_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_buf_cb cb, uint64_t len)
{
struct spdk_bdev *bdev = bdev_io->bdev;
uint64_t alignment;
assert(cb != NULL);
bdev_io->internal.get_buf_cb = cb;
alignment = spdk_bdev_get_buf_align(bdev);
if (_is_buf_allocated(bdev_io->u.bdev.iovs) &&
_are_iovs_aligned(bdev_io->u.bdev.iovs, bdev_io->u.bdev.iovcnt, alignment)) {
/* Buffer already present and aligned */
cb(spdk_bdev_io_get_io_channel(bdev_io), bdev_io, true);
return;
}
bdev_io_get_buf(bdev_io, len);
}
void
spdk_bdev_io_get_aux_buf(struct spdk_bdev_io *bdev_io, spdk_bdev_io_get_aux_buf_cb cb)
{
uint64_t len = bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
assert(cb != NULL);
assert(bdev_io->internal.get_aux_buf_cb == NULL);
bdev_io->internal.get_aux_buf_cb = cb;
bdev_io_get_buf(bdev_io, len);
}
static int
bdev_module_get_max_ctx_size(void)
{
struct spdk_bdev_module *bdev_module;
int max_bdev_module_size = 0;
TAILQ_FOREACH(bdev_module, &g_bdev_mgr.bdev_modules, internal.tailq) {
if (bdev_module->get_ctx_size && bdev_module->get_ctx_size() > max_bdev_module_size) {
max_bdev_module_size = bdev_module->get_ctx_size();
}
}
return max_bdev_module_size;
}
static void
bdev_qos_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
{
int i;
struct spdk_bdev_qos *qos = bdev->internal.qos;
uint64_t limits[SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES];
if (!qos) {
return;
}
spdk_bdev_get_qos_rate_limits(bdev, limits);
spdk_json_write_object_begin(w);
spdk_json_write_named_string(w, "method", "bdev_set_qos_limit");
spdk_json_write_named_object_begin(w, "params");
spdk_json_write_named_string(w, "name", bdev->name);
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
if (limits[i] > 0) {
spdk_json_write_named_uint64(w, qos_rpc_type[i], limits[i]);
}
}
spdk_json_write_object_end(w);
spdk_json_write_object_end(w);
}
void
spdk_bdev_subsystem_config_json(struct spdk_json_write_ctx *w)
{
struct spdk_bdev_module *bdev_module;
struct spdk_bdev *bdev;
assert(w != NULL);
spdk_json_write_array_begin(w);
spdk_json_write_object_begin(w);
spdk_json_write_named_string(w, "method", "bdev_set_options");
spdk_json_write_named_object_begin(w, "params");
spdk_json_write_named_uint32(w, "bdev_io_pool_size", g_bdev_opts.bdev_io_pool_size);
spdk_json_write_named_uint32(w, "bdev_io_cache_size", g_bdev_opts.bdev_io_cache_size);
spdk_json_write_named_bool(w, "bdev_auto_examine", g_bdev_opts.bdev_auto_examine);
spdk_json_write_object_end(w);
spdk_json_write_object_end(w);
bdev_examine_allowlist_config_json(w);
TAILQ_FOREACH(bdev_module, &g_bdev_mgr.bdev_modules, internal.tailq) {
if (bdev_module->config_json) {
bdev_module->config_json(w);
}
}
pthread_mutex_lock(&g_bdev_mgr.mutex);
TAILQ_FOREACH(bdev, &g_bdev_mgr.bdevs, internal.link) {
if (bdev->fn_table->write_config_json) {
bdev->fn_table->write_config_json(bdev, w);
}
bdev_qos_config_json(bdev, w);
}
pthread_mutex_unlock(&g_bdev_mgr.mutex);
/* This has to be last RPC in array to make sure all bdevs finished examine */
spdk_json_write_object_begin(w);
spdk_json_write_named_string(w, "method", "bdev_wait_for_examine");
spdk_json_write_object_end(w);
spdk_json_write_array_end(w);
}
static int
bdev_mgmt_channel_create(void *io_device, void *ctx_buf)
{
struct spdk_bdev_mgmt_channel *ch = ctx_buf;
struct spdk_bdev_io *bdev_io;
uint32_t i;
STAILQ_INIT(&ch->need_buf_small);
STAILQ_INIT(&ch->need_buf_large);
STAILQ_INIT(&ch->per_thread_cache);
ch->bdev_io_cache_size = g_bdev_opts.bdev_io_cache_size;
/* Pre-populate bdev_io cache to ensure this thread cannot be starved. */
ch->per_thread_cache_count = 0;
for (i = 0; i < ch->bdev_io_cache_size; i++) {
bdev_io = spdk_mempool_get(g_bdev_mgr.bdev_io_pool);
assert(bdev_io != NULL);
ch->per_thread_cache_count++;
STAILQ_INSERT_HEAD(&ch->per_thread_cache, bdev_io, internal.buf_link);
}
TAILQ_INIT(&ch->shared_resources);
TAILQ_INIT(&ch->io_wait_queue);
return 0;
}
static void
bdev_mgmt_channel_destroy(void *io_device, void *ctx_buf)
{
struct spdk_bdev_mgmt_channel *ch = ctx_buf;
struct spdk_bdev_io *bdev_io;
if (!STAILQ_EMPTY(&ch->need_buf_small) || !STAILQ_EMPTY(&ch->need_buf_large)) {
SPDK_ERRLOG("Pending I/O list wasn't empty on mgmt channel free\n");
}
if (!TAILQ_EMPTY(&ch->shared_resources)) {
SPDK_ERRLOG("Module channel list wasn't empty on mgmt channel free\n");
}
while (!STAILQ_EMPTY(&ch->per_thread_cache)) {
bdev_io = STAILQ_FIRST(&ch->per_thread_cache);
STAILQ_REMOVE_HEAD(&ch->per_thread_cache, internal.buf_link);
ch->per_thread_cache_count--;
spdk_mempool_put(g_bdev_mgr.bdev_io_pool, (void *)bdev_io);
}
assert(ch->per_thread_cache_count == 0);
}
static void
bdev_init_complete(int rc)
{
spdk_bdev_init_cb cb_fn = g_init_cb_fn;
void *cb_arg = g_init_cb_arg;
struct spdk_bdev_module *m;
g_bdev_mgr.init_complete = true;
g_init_cb_fn = NULL;
g_init_cb_arg = NULL;
/*
* For modules that need to know when subsystem init is complete,
* inform them now.
*/
if (rc == 0) {
TAILQ_FOREACH(m, &g_bdev_mgr.bdev_modules, internal.tailq) {
if (m->init_complete) {
m->init_complete();
}
}
}
cb_fn(cb_arg, rc);
}
static bool
bdev_module_all_actions_completed(void)
{
struct spdk_bdev_module *m;
TAILQ_FOREACH(m, &g_bdev_mgr.bdev_modules, internal.tailq) {
if (m->internal.action_in_progress > 0) {
return false;
}
}
return true;
}
static void
bdev_module_action_complete(void)
{
/*
* Don't finish bdev subsystem initialization if
* module pre-initialization is still in progress, or
* the subsystem been already initialized.
*/
if (!g_bdev_mgr.module_init_complete || g_bdev_mgr.init_complete) {
return;
}
/*
* Check all bdev modules for inits/examinations in progress. If any
* exist, return immediately since we cannot finish bdev subsystem
* initialization until all are completed.
*/
if (!bdev_module_all_actions_completed()) {
return;
}
/*
* Modules already finished initialization - now that all
* the bdev modules have finished their asynchronous I/O
* processing, the entire bdev layer can be marked as complete.
*/
bdev_init_complete(0);
}
static void
bdev_module_action_done(struct spdk_bdev_module *module)
{
assert(module->internal.action_in_progress > 0);
module->internal.action_in_progress--;
bdev_module_action_complete();
}
void
spdk_bdev_module_init_done(struct spdk_bdev_module *module)
{
bdev_module_action_done(module);
}
void
spdk_bdev_module_examine_done(struct spdk_bdev_module *module)
{
bdev_module_action_done(module);
}
/** The last initialized bdev module */
static struct spdk_bdev_module *g_resume_bdev_module = NULL;
static void
bdev_init_failed(void *cb_arg)
{
struct spdk_bdev_module *module = cb_arg;
module->internal.action_in_progress--;
bdev_init_complete(-1);
}
static int
bdev_modules_init(void)
{
struct spdk_bdev_module *module;
int rc = 0;
TAILQ_FOREACH(module, &g_bdev_mgr.bdev_modules, internal.tailq) {
g_resume_bdev_module = module;
if (module->async_init) {
module->internal.action_in_progress = 1;
}
rc = module->module_init();
if (rc != 0) {
/* Bump action_in_progress to prevent other modules from completion of modules_init
* Send message to defer application shutdown until resources are cleaned up */
module->internal.action_in_progress = 1;
spdk_thread_send_msg(spdk_get_thread(), bdev_init_failed, module);
return rc;
}
}
g_resume_bdev_module = NULL;
return 0;
}
void
spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg)
{
int cache_size;
int rc = 0;
char mempool_name[32];
assert(cb_fn != NULL);
g_init_cb_fn = cb_fn;
g_init_cb_arg = cb_arg;
spdk_notify_type_register("bdev_register");
spdk_notify_type_register("bdev_unregister");
snprintf(mempool_name, sizeof(mempool_name), "bdev_io_%d", getpid());
g_bdev_mgr.bdev_io_pool = spdk_mempool_create(mempool_name,
g_bdev_opts.bdev_io_pool_size,
sizeof(struct spdk_bdev_io) +
bdev_module_get_max_ctx_size(),
0,
SPDK_ENV_SOCKET_ID_ANY);
if (g_bdev_mgr.bdev_io_pool == NULL) {
SPDK_ERRLOG("could not allocate spdk_bdev_io pool\n");
bdev_init_complete(-1);
return;
}
/**
* Ensure no more than half of the total buffers end up local caches, by
* using spdk_env_get_core_count() to determine how many local caches we need
* to account for.
*/
cache_size = BUF_SMALL_POOL_SIZE / (2 * spdk_env_get_core_count());
snprintf(mempool_name, sizeof(mempool_name), "buf_small_pool_%d", getpid());
g_bdev_mgr.buf_small_pool = spdk_mempool_create(mempool_name,
g_bdev_opts.small_buf_pool_size,
SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_SMALL_BUF_MAX_SIZE) +
SPDK_BDEV_POOL_ALIGNMENT,
cache_size,
SPDK_ENV_SOCKET_ID_ANY);
if (!g_bdev_mgr.buf_small_pool) {
SPDK_ERRLOG("create rbuf small pool failed\n");
bdev_init_complete(-1);
return;
}
cache_size = BUF_LARGE_POOL_SIZE / (2 * spdk_env_get_core_count());
snprintf(mempool_name, sizeof(mempool_name), "buf_large_pool_%d", getpid());
g_bdev_mgr.buf_large_pool = spdk_mempool_create(mempool_name,
g_bdev_opts.large_buf_pool_size,
SPDK_BDEV_BUF_SIZE_WITH_MD(SPDK_BDEV_LARGE_BUF_MAX_SIZE) +
SPDK_BDEV_POOL_ALIGNMENT,
cache_size,
SPDK_ENV_SOCKET_ID_ANY);
if (!g_bdev_mgr.buf_large_pool) {
SPDK_ERRLOG("create rbuf large pool failed\n");
bdev_init_complete(-1);
return;
}
g_bdev_mgr.zero_buffer = spdk_zmalloc(ZERO_BUFFER_SIZE, ZERO_BUFFER_SIZE,
NULL, SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
if (!g_bdev_mgr.zero_buffer) {
SPDK_ERRLOG("create bdev zero buffer failed\n");
bdev_init_complete(-1);
return;
}
#ifdef SPDK_CONFIG_VTUNE
g_bdev_mgr.domain = __itt_domain_create("spdk_bdev");
#endif
spdk_io_device_register(&g_bdev_mgr, bdev_mgmt_channel_create,
bdev_mgmt_channel_destroy,
sizeof(struct spdk_bdev_mgmt_channel),
"bdev_mgr");
rc = bdev_modules_init();
g_bdev_mgr.module_init_complete = true;
if (rc != 0) {
SPDK_ERRLOG("bdev modules init failed\n");
return;
}
bdev_module_action_complete();
}
static void
bdev_mgr_unregister_cb(void *io_device)
{
spdk_bdev_fini_cb cb_fn = g_fini_cb_fn;
if (g_bdev_mgr.bdev_io_pool) {
if (spdk_mempool_count(g_bdev_mgr.bdev_io_pool) != g_bdev_opts.bdev_io_pool_size) {
SPDK_ERRLOG("bdev IO pool count is %zu but should be %u\n",
spdk_mempool_count(g_bdev_mgr.bdev_io_pool),
g_bdev_opts.bdev_io_pool_size);
}
spdk_mempool_free(g_bdev_mgr.bdev_io_pool);
}
if (g_bdev_mgr.buf_small_pool) {
if (spdk_mempool_count(g_bdev_mgr.buf_small_pool) != g_bdev_opts.small_buf_pool_size) {
SPDK_ERRLOG("Small buffer pool count is %zu but should be %u\n",
spdk_mempool_count(g_bdev_mgr.buf_small_pool),
g_bdev_opts.small_buf_pool_size);
assert(false);
}
spdk_mempool_free(g_bdev_mgr.buf_small_pool);
}
if (g_bdev_mgr.buf_large_pool) {
if (spdk_mempool_count(g_bdev_mgr.buf_large_pool) != g_bdev_opts.large_buf_pool_size) {
SPDK_ERRLOG("Large buffer pool count is %zu but should be %u\n",
spdk_mempool_count(g_bdev_mgr.buf_large_pool),
g_bdev_opts.large_buf_pool_size);
assert(false);
}
spdk_mempool_free(g_bdev_mgr.buf_large_pool);
}
spdk_free(g_bdev_mgr.zero_buffer);
bdev_examine_allowlist_free();
cb_fn(g_fini_cb_arg);
g_fini_cb_fn = NULL;
g_fini_cb_arg = NULL;
g_bdev_mgr.init_complete = false;
g_bdev_mgr.module_init_complete = false;
}
static void
bdev_module_finish_iter(void *arg)
{
struct spdk_bdev_module *bdev_module;
/* FIXME: Handling initialization failures is broken now,
* so we won't even try cleaning up after successfully
* initialized modules. if module_init_complete is false,
* just call spdk_bdev_mgr_unregister_cb
*/
if (!g_bdev_mgr.module_init_complete) {
bdev_mgr_unregister_cb(NULL);
return;
}
/* Start iterating from the last touched module */
if (!g_resume_bdev_module) {
bdev_module = TAILQ_LAST(&g_bdev_mgr.bdev_modules, bdev_module_list);
} else {
bdev_module = TAILQ_PREV(g_resume_bdev_module, bdev_module_list,
internal.tailq);
}
while (bdev_module) {
if (bdev_module->async_fini) {
/* Save our place so we can resume later. We must
* save the variable here, before calling module_fini()
* below, because in some cases the module may immediately
* call spdk_bdev_module_finish_done() and re-enter
* this function to continue iterating. */
g_resume_bdev_module = bdev_module;
}
if (bdev_module->module_fini) {
bdev_module->module_fini();
}
if (bdev_module->async_fini) {
return;
}
bdev_module = TAILQ_PREV(bdev_module, bdev_module_list,
internal.tailq);
}
g_resume_bdev_module = NULL;
spdk_io_device_unregister(&g_bdev_mgr, bdev_mgr_unregister_cb);
}
void
spdk_bdev_module_finish_done(void)
{
if (spdk_get_thread() != g_fini_thread) {
spdk_thread_send_msg(g_fini_thread, bdev_module_finish_iter, NULL);
} else {
bdev_module_finish_iter(NULL);
}
}
static void
bdev_finish_unregister_bdevs_iter(void *cb_arg, int bdeverrno)
{
struct spdk_bdev *bdev = cb_arg;
if (bdeverrno && bdev) {
SPDK_WARNLOG("Unable to unregister bdev '%s' during spdk_bdev_finish()\n",
bdev->name);
/*
* Since the call to spdk_bdev_unregister() failed, we have no way to free this
* bdev; try to continue by manually removing this bdev from the list and continue
* with the next bdev in the list.
*/
TAILQ_REMOVE(&g_bdev_mgr.bdevs, bdev, internal.link);
}
if (TAILQ_EMPTY(&g_bdev_mgr.bdevs)) {
SPDK_DEBUGLOG(bdev, "Done unregistering bdevs\n");
/*
* Bdev module finish need to be deferred as we might be in the middle of some context
* (like bdev part free) that will use this bdev (or private bdev driver ctx data)
* after returning.
*/
spdk_thread_send_msg(spdk_get_thread(), bdev_module_finish_iter, NULL);
return;
}
/*
* Unregister last unclaimed bdev in the list, to ensure that bdev subsystem
* shutdown proceeds top-down. The goal is to give virtual bdevs an opportunity
* to detect clean shutdown as opposed to run-time hot removal of the underlying
* base bdevs.
*
* Also, walk the list in the reverse order.
*/
for (bdev = TAILQ_LAST(&g_bdev_mgr.bdevs, spdk_bdev_list);
bdev; bdev = TAILQ_PREV(bdev, spdk_bdev_list, internal.link)) {
if (bdev->internal.claim_module != NULL) {
SPDK_DEBUGLOG(bdev, "Skipping claimed bdev '%s'(<-'%s').\n",
bdev->name, bdev->internal.claim_module->name);
continue;
}
SPDK_DEBUGLOG(bdev, "Unregistering bdev '%s'\n", bdev->name);
spdk_bdev_unregister(bdev, bdev_finish_unregister_bdevs_iter, bdev);
return;
}
/*
* If any bdev fails to unclaim underlying bdev properly, we may face the
* case of bdev list consisting of claimed bdevs only (if claims are managed
* correctly, this would mean there's a loop in the claims graph which is
* clearly impossible). Warn and unregister last bdev on the list then.
*/
for (bdev = TAILQ_LAST(&g_bdev_mgr.bdevs, spdk_bdev_list);
bdev; bdev = TAILQ_PREV(bdev, spdk_bdev_list, internal.link)) {
SPDK_WARNLOG("Unregistering claimed bdev '%s'!\n", bdev->name);
spdk_bdev_unregister(bdev, bdev_finish_unregister_bdevs_iter, bdev);
return;
}
}
void
spdk_bdev_finish(spdk_bdev_fini_cb cb_fn, void *cb_arg)
{
struct spdk_bdev_module *m;
assert(cb_fn != NULL);
g_fini_thread = spdk_get_thread();
g_fini_cb_fn = cb_fn;
g_fini_cb_arg = cb_arg;
TAILQ_FOREACH(m, &g_bdev_mgr.bdev_modules, internal.tailq) {
if (m->fini_start) {
m->fini_start();
}
}
bdev_finish_unregister_bdevs_iter(NULL, 0);
}
struct spdk_bdev_io *
bdev_channel_get_io(struct spdk_bdev_channel *channel)
{
struct spdk_bdev_mgmt_channel *ch = channel->shared_resource->mgmt_ch;
struct spdk_bdev_io *bdev_io;
if (ch->per_thread_cache_count > 0) {
bdev_io = STAILQ_FIRST(&ch->per_thread_cache);
STAILQ_REMOVE_HEAD(&ch->per_thread_cache, internal.buf_link);
ch->per_thread_cache_count--;
} else if (spdk_unlikely(!TAILQ_EMPTY(&ch->io_wait_queue))) {
/*
* Don't try to look for bdev_ios in the global pool if there are
* waiters on bdev_ios - we don't want this caller to jump the line.
*/
bdev_io = NULL;
} else {
bdev_io = spdk_mempool_get(g_bdev_mgr.bdev_io_pool);
}
return bdev_io;
}
void
spdk_bdev_free_io(struct spdk_bdev_io *bdev_io)
{
struct spdk_bdev_mgmt_channel *ch;
assert(bdev_io != NULL);
assert(bdev_io->internal.status != SPDK_BDEV_IO_STATUS_PENDING);
ch = bdev_io->internal.ch->shared_resource->mgmt_ch;
if (bdev_io->internal.buf != NULL) {
bdev_io_put_buf(bdev_io);
}
if (ch->per_thread_cache_count < ch->bdev_io_cache_size) {
ch->per_thread_cache_count++;
STAILQ_INSERT_HEAD(&ch->per_thread_cache, bdev_io, internal.buf_link);
while (ch->per_thread_cache_count > 0 && !TAILQ_EMPTY(&ch->io_wait_queue)) {
struct spdk_bdev_io_wait_entry *entry;
entry = TAILQ_FIRST(&ch->io_wait_queue);
TAILQ_REMOVE(&ch->io_wait_queue, entry, link);
entry->cb_fn(entry->cb_arg);
}
} else {
/* We should never have a full cache with entries on the io wait queue. */
assert(TAILQ_EMPTY(&ch->io_wait_queue));
spdk_mempool_put(g_bdev_mgr.bdev_io_pool, (void *)bdev_io);
}
}
static bool
bdev_qos_is_iops_rate_limit(enum spdk_bdev_qos_rate_limit_type limit)
{
assert(limit != SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES);
switch (limit) {
case SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT:
return true;
case SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT:
case SPDK_BDEV_QOS_R_BPS_RATE_LIMIT:
case SPDK_BDEV_QOS_W_BPS_RATE_LIMIT:
return false;
case SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES:
default:
return false;
}
}
static bool
bdev_qos_io_to_limit(struct spdk_bdev_io *bdev_io)
{
switch (bdev_io->type) {
case SPDK_BDEV_IO_TYPE_NVME_IO:
case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
case SPDK_BDEV_IO_TYPE_READ:
case SPDK_BDEV_IO_TYPE_WRITE:
return true;
case SPDK_BDEV_IO_TYPE_ZCOPY:
if (bdev_io->u.bdev.zcopy.start) {
return true;
} else {
return false;
}
default:
return false;
}
}
static bool
bdev_is_read_io(struct spdk_bdev_io *bdev_io)
{
switch (bdev_io->type) {
case SPDK_BDEV_IO_TYPE_NVME_IO:
case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
/* Bit 1 (0x2) set for read operation */
if (bdev_io->u.nvme_passthru.cmd.opc & SPDK_NVME_OPC_READ) {
return true;
} else {
return false;
}
case SPDK_BDEV_IO_TYPE_READ:
return true;
case SPDK_BDEV_IO_TYPE_ZCOPY:
/* Populate to read from disk */
if (bdev_io->u.bdev.zcopy.populate) {
return true;
} else {
return false;
}
default:
return false;
}
}
static uint64_t
bdev_get_io_size_in_byte(struct spdk_bdev_io *bdev_io)
{
struct spdk_bdev *bdev = bdev_io->bdev;
switch (bdev_io->type) {
case SPDK_BDEV_IO_TYPE_NVME_IO:
case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
return bdev_io->u.nvme_passthru.nbytes;
case SPDK_BDEV_IO_TYPE_READ:
case SPDK_BDEV_IO_TYPE_WRITE:
return bdev_io->u.bdev.num_blocks * bdev->blocklen;
case SPDK_BDEV_IO_TYPE_ZCOPY:
/* Track the data in the start phase only */
if (bdev_io->u.bdev.zcopy.start) {
return bdev_io->u.bdev.num_blocks * bdev->blocklen;
} else {
return 0;
}
default:
return 0;
}
}
static bool
bdev_qos_rw_queue_io(const struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
{
if (limit->max_per_timeslice > 0 && limit->remaining_this_timeslice <= 0) {
return true;
} else {
return false;
}
}
static bool
bdev_qos_r_queue_io(const struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
{
if (bdev_is_read_io(io) == false) {
return false;
}
return bdev_qos_rw_queue_io(limit, io);
}
static bool
bdev_qos_w_queue_io(const struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
{
if (bdev_is_read_io(io) == true) {
return false;
}
return bdev_qos_rw_queue_io(limit, io);
}
static void
bdev_qos_rw_iops_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
{
limit->remaining_this_timeslice--;
}
static void
bdev_qos_rw_bps_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
{
limit->remaining_this_timeslice -= bdev_get_io_size_in_byte(io);
}
static void
bdev_qos_r_bps_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
{
if (bdev_is_read_io(io) == false) {
return;
}
return bdev_qos_rw_bps_update_quota(limit, io);
}
static void
bdev_qos_w_bps_update_quota(struct spdk_bdev_qos_limit *limit, struct spdk_bdev_io *io)
{
if (bdev_is_read_io(io) == true) {
return;
}
return bdev_qos_rw_bps_update_quota(limit, io);
}
static void
bdev_qos_set_ops(struct spdk_bdev_qos *qos)
{
int i;
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
if (qos->rate_limits[i].limit == SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {
qos->rate_limits[i].queue_io = NULL;
qos->rate_limits[i].update_quota = NULL;
continue;
}
switch (i) {
case SPDK_BDEV_QOS_RW_IOPS_RATE_LIMIT:
qos->rate_limits[i].queue_io = bdev_qos_rw_queue_io;
qos->rate_limits[i].update_quota = bdev_qos_rw_iops_update_quota;
break;
case SPDK_BDEV_QOS_RW_BPS_RATE_LIMIT:
qos->rate_limits[i].queue_io = bdev_qos_rw_queue_io;
qos->rate_limits[i].update_quota = bdev_qos_rw_bps_update_quota;
break;
case SPDK_BDEV_QOS_R_BPS_RATE_LIMIT:
qos->rate_limits[i].queue_io = bdev_qos_r_queue_io;
qos->rate_limits[i].update_quota = bdev_qos_r_bps_update_quota;
break;
case SPDK_BDEV_QOS_W_BPS_RATE_LIMIT:
qos->rate_limits[i].queue_io = bdev_qos_w_queue_io;
qos->rate_limits[i].update_quota = bdev_qos_w_bps_update_quota;
break;
default:
break;
}
}
}
static void
_bdev_io_complete_in_submit(struct spdk_bdev_channel *bdev_ch,
struct spdk_bdev_io *bdev_io,
enum spdk_bdev_io_status status)
{
struct spdk_bdev_shared_resource *shared_resource = bdev_ch->shared_resource;
bdev_io->internal.in_submit_request = true;
bdev_ch->io_outstanding++;
shared_resource->io_outstanding++;
spdk_bdev_io_complete(bdev_io, status);
bdev_io->internal.in_submit_request = false;
}
static inline void
bdev_io_do_submit(struct spdk_bdev_channel *bdev_ch, struct spdk_bdev_io *bdev_io)
{
struct spdk_bdev *bdev = bdev_io->bdev;
struct spdk_io_channel *ch = bdev_ch->channel;
struct spdk_bdev_shared_resource *shared_resource = bdev_ch->shared_resource;
if (spdk_unlikely(bdev_io->type == SPDK_BDEV_IO_TYPE_ABORT)) {
struct spdk_bdev_mgmt_channel *mgmt_channel = shared_resource->mgmt_ch;
struct spdk_bdev_io *bio_to_abort = bdev_io->u.abort.bio_to_abort;
if (bdev_abort_queued_io(&shared_resource->nomem_io, bio_to_abort) ||
bdev_abort_buf_io(&mgmt_channel->need_buf_small, bio_to_abort) ||
bdev_abort_buf_io(&mgmt_channel->need_buf_large, bio_to_abort)) {
_bdev_io_complete_in_submit(bdev_ch, bdev_io,
SPDK_BDEV_IO_STATUS_SUCCESS);
return;
}
}
if (spdk_likely(TAILQ_EMPTY(&shared_resource->nomem_io))) {
bdev_ch->io_outstanding++;
shared_resource->io_outstanding++;
bdev_io->internal.in_submit_request = true;
bdev->fn_table->submit_request(ch, bdev_io);
bdev_io->internal.in_submit_request = false;
} else {
TAILQ_INSERT_TAIL(&shared_resource->nomem_io, bdev_io, internal.link);
}
}
static int
bdev_qos_io_submit(struct spdk_bdev_channel *ch, struct spdk_bdev_qos *qos)
{
struct spdk_bdev_io *bdev_io = NULL, *tmp = NULL;
int i, submitted_ios = 0;
TAILQ_FOREACH_SAFE(bdev_io, &qos->queued, internal.link, tmp) {
if (bdev_qos_io_to_limit(bdev_io) == true) {
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
if (!qos->rate_limits[i].queue_io) {
continue;
}
if (qos->rate_limits[i].queue_io(&qos->rate_limits[i],
bdev_io) == true) {
return submitted_ios;
}
}
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
if (!qos->rate_limits[i].update_quota) {
continue;
}
qos->rate_limits[i].update_quota(&qos->rate_limits[i], bdev_io);
}
}
TAILQ_REMOVE(&qos->queued, bdev_io, internal.link);
bdev_io_do_submit(ch, bdev_io);
submitted_ios++;
}
return submitted_ios;
}
static void
bdev_queue_io_wait_with_cb(struct spdk_bdev_io *bdev_io, spdk_bdev_io_wait_cb cb_fn)
{
int rc;
bdev_io->internal.waitq_entry.bdev = bdev_io->bdev;
bdev_io->internal.waitq_entry.cb_fn = cb_fn;
bdev_io->internal.waitq_entry.cb_arg = bdev_io;
rc = spdk_bdev_queue_io_wait(bdev_io->bdev, spdk_io_channel_from_ctx(bdev_io->internal.ch),
&bdev_io->internal.waitq_entry);
if (rc != 0) {
SPDK_ERRLOG("Queue IO failed, rc=%d\n", rc);
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
}
}
static bool
bdev_rw_should_split(struct spdk_bdev_io *bdev_io)
{
uint32_t io_boundary = bdev_io->bdev->optimal_io_boundary;
uint32_t max_size = bdev_io->bdev->max_segment_size;
int max_segs = bdev_io->bdev->max_num_segments;
io_boundary = bdev_io->bdev->split_on_optimal_io_boundary ? io_boundary : 0;
if (spdk_likely(!io_boundary && !max_segs && !max_size)) {
return false;
}
if (io_boundary) {
uint64_t start_stripe, end_stripe;
start_stripe = bdev_io->u.bdev.offset_blocks;
end_stripe = start_stripe + bdev_io->u.bdev.num_blocks - 1;
/* Avoid expensive div operations if possible. These spdk_u32 functions are very cheap. */
if (spdk_likely(spdk_u32_is_pow2(io_boundary))) {
start_stripe >>= spdk_u32log2(io_boundary);
end_stripe >>= spdk_u32log2(io_boundary);
} else {
start_stripe /= io_boundary;
end_stripe /= io_boundary;
}
if (start_stripe != end_stripe) {
return true;
}
}
if (max_segs) {
if (bdev_io->u.bdev.iovcnt > max_segs) {
return true;
}
}
if (max_size) {
for (int i = 0; i < bdev_io->u.bdev.iovcnt; i++) {
if (bdev_io->u.bdev.iovs[i].iov_len > max_size) {
return true;
}
}
}
return false;
}
static bool
bdev_unmap_should_split(struct spdk_bdev_io *bdev_io)
{
uint32_t num_unmap_segments;
if (!bdev_io->bdev->max_unmap || !bdev_io->bdev->max_unmap_segments) {
return false;
}
num_unmap_segments = spdk_divide_round_up(bdev_io->u.bdev.num_blocks, bdev_io->bdev->max_unmap);
if (num_unmap_segments > bdev_io->bdev->max_unmap_segments) {
return true;
}
return false;
}
static bool
bdev_write_zeroes_should_split(struct spdk_bdev_io *bdev_io)
{
if (!bdev_io->bdev->max_write_zeroes) {
return false;
}
if (bdev_io->u.bdev.num_blocks > bdev_io->bdev->max_write_zeroes) {
return true;
}
return false;
}
static bool
bdev_io_should_split(struct spdk_bdev_io *bdev_io)
{
switch (bdev_io->type) {
case SPDK_BDEV_IO_TYPE_READ:
case SPDK_BDEV_IO_TYPE_WRITE:
return bdev_rw_should_split(bdev_io);
case SPDK_BDEV_IO_TYPE_UNMAP:
return bdev_unmap_should_split(bdev_io);
case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
return bdev_write_zeroes_should_split(bdev_io);
default:
return false;
}
}
static uint32_t
_to_next_boundary(uint64_t offset, uint32_t boundary)
{
return (boundary - (offset % boundary));
}
static void
bdev_io_split_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg);
static void
_bdev_rw_split(void *_bdev_io);
static void
bdev_unmap_split(struct spdk_bdev_io *bdev_io);
static void
_bdev_unmap_split(void *_bdev_io)
{
return bdev_unmap_split((struct spdk_bdev_io *)_bdev_io);
}
static void
bdev_write_zeroes_split(struct spdk_bdev_io *bdev_io);
static void
_bdev_write_zeroes_split(void *_bdev_io)
{
return bdev_write_zeroes_split((struct spdk_bdev_io *)_bdev_io);
}
static int
bdev_io_split_submit(struct spdk_bdev_io *bdev_io, struct iovec *iov, int iovcnt, void *md_buf,
uint64_t num_blocks, uint64_t *offset, uint64_t *remaining)
{
int rc;
uint64_t current_offset, current_remaining;
spdk_bdev_io_wait_cb io_wait_fn;
current_offset = *offset;
current_remaining = *remaining;
bdev_io->u.bdev.split_outstanding++;
io_wait_fn = _bdev_rw_split;
switch (bdev_io->type) {
case SPDK_BDEV_IO_TYPE_READ:
rc = bdev_readv_blocks_with_md(bdev_io->internal.desc,
spdk_io_channel_from_ctx(bdev_io->internal.ch),
iov, iovcnt, md_buf, current_offset,
num_blocks,
bdev_io_split_done, bdev_io);
break;
case SPDK_BDEV_IO_TYPE_WRITE:
rc = bdev_writev_blocks_with_md(bdev_io->internal.desc,
spdk_io_channel_from_ctx(bdev_io->internal.ch),
iov, iovcnt, md_buf, current_offset,
num_blocks,
bdev_io_split_done, bdev_io);
break;
case SPDK_BDEV_IO_TYPE_UNMAP:
io_wait_fn = _bdev_unmap_split;
rc = spdk_bdev_unmap_blocks(bdev_io->internal.desc,
spdk_io_channel_from_ctx(bdev_io->internal.ch),
current_offset, num_blocks,
bdev_io_split_done, bdev_io);
break;
case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
io_wait_fn = _bdev_write_zeroes_split;
rc = spdk_bdev_write_zeroes_blocks(bdev_io->internal.desc,
spdk_io_channel_from_ctx(bdev_io->internal.ch),
current_offset, num_blocks,
bdev_io_split_done, bdev_io);
break;
default:
assert(false);
rc = -EINVAL;
break;
}
if (rc == 0) {
current_offset += num_blocks;
current_remaining -= num_blocks;
bdev_io->u.bdev.split_current_offset_blocks = current_offset;
bdev_io->u.bdev.split_remaining_num_blocks = current_remaining;
*offset = current_offset;
*remaining = current_remaining;
} else {
bdev_io->u.bdev.split_outstanding--;
if (rc == -ENOMEM) {
if (bdev_io->u.bdev.split_outstanding == 0) {
/* No I/O is outstanding. Hence we should wait here. */
bdev_queue_io_wait_with_cb(bdev_io, io_wait_fn);
}
} else {
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
if (bdev_io->u.bdev.split_outstanding == 0) {
spdk_trace_record(TRACE_BDEV_IO_DONE, 0, 0, (uintptr_t)bdev_io);
TAILQ_REMOVE(&bdev_io->internal.ch->io_submitted, bdev_io, internal.ch_link);
bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
}
}
}
return rc;
}
static void
_bdev_rw_split(void *_bdev_io)
{
struct iovec *parent_iov, *iov;
struct spdk_bdev_io *bdev_io = _bdev_io;
struct spdk_bdev *bdev = bdev_io->bdev;
uint64_t parent_offset, current_offset, remaining;
uint32_t parent_iov_offset, parent_iovcnt, parent_iovpos, child_iovcnt;
uint32_t to_next_boundary, to_next_boundary_bytes, to_last_block_bytes;
uint32_t iovcnt, iov_len, child_iovsize;
uint32_t blocklen = bdev->blocklen;
uint32_t io_boundary = bdev->optimal_io_boundary;
uint32_t max_segment_size = bdev->max_segment_size;
uint32_t max_child_iovcnt = bdev->max_num_segments;
void *md_buf = NULL;
int rc;
max_segment_size = max_segment_size ? max_segment_size : UINT32_MAX;
max_child_iovcnt = max_child_iovcnt ? spdk_min(max_child_iovcnt, BDEV_IO_NUM_CHILD_IOV) :
BDEV_IO_NUM_CHILD_IOV;
io_boundary = bdev->split_on_optimal_io_boundary ? io_boundary : UINT32_MAX;
remaining = bdev_io->u.bdev.split_remaining_num_blocks;
current_offset = bdev_io->u.bdev.split_current_offset_blocks;
parent_offset = bdev_io->u.bdev.offset_blocks;
parent_iov_offset = (current_offset - parent_offset) * blocklen;
parent_iovcnt = bdev_io->u.bdev.iovcnt;
for (parent_iovpos = 0; parent_iovpos < parent_iovcnt; parent_iovpos++) {
parent_iov = &bdev_io->u.bdev.iovs[parent_iovpos];
if (parent_iov_offset < parent_iov->iov_len) {
break;
}
parent_iov_offset -= parent_iov->iov_len;
}
child_iovcnt = 0;
while (remaining > 0 && parent_iovpos < parent_iovcnt && child_iovcnt < BDEV_IO_NUM_CHILD_IOV) {
to_next_boundary = _to_next_boundary(current_offset, io_boundary);
to_next_boundary = spdk_min(remaining, to_next_boundary);
to_next_boundary_bytes = to_next_boundary * blocklen;
iov = &bdev_io->child_iov[child_iovcnt];
iovcnt = 0;
if (bdev_io->u.bdev.md_buf) {
md_buf = (char *)bdev_io->u.bdev.md_buf +
(current_offset - parent_offset) * spdk_bdev_get_md_size(bdev);
}
child_iovsize = spdk_min(BDEV_IO_NUM_CHILD_IOV - child_iovcnt, max_child_iovcnt);
while (to_next_boundary_bytes > 0 && parent_iovpos < parent_iovcnt &&
iovcnt < child_iovsize) {
parent_iov = &bdev_io->u.bdev.iovs[parent_iovpos];
iov_len = parent_iov->iov_len - parent_iov_offset;
iov_len = spdk_min(iov_len, max_segment_size);
iov_len = spdk_min(iov_len, to_next_boundary_bytes);
to_next_boundary_bytes -= iov_len;
bdev_io->child_iov[child_iovcnt].iov_base = parent_iov->iov_base + parent_iov_offset;
bdev_io->child_iov[child_iovcnt].iov_len = iov_len;
if (iov_len < parent_iov->iov_len - parent_iov_offset) {
parent_iov_offset += iov_len;
} else {
parent_iovpos++;
parent_iov_offset = 0;
}
child_iovcnt++;
iovcnt++;
}
if (to_next_boundary_bytes > 0) {
/* We had to stop this child I/O early because we ran out of
* child_iov space or were limited by max_num_segments.
* Ensure the iovs to be aligned with block size and
* then adjust to_next_boundary before starting the
* child I/O.
*/
assert(child_iovcnt == BDEV_IO_NUM_CHILD_IOV ||
iovcnt == child_iovsize);
to_last_block_bytes = to_next_boundary_bytes % blocklen;
if (to_last_block_bytes != 0) {
uint32_t child_iovpos = child_iovcnt - 1;
/* don't decrease child_iovcnt when it equals to BDEV_IO_NUM_CHILD_IOV
* so the loop will naturally end
*/
to_last_block_bytes = blocklen - to_last_block_bytes;
to_next_boundary_bytes += to_last_block_bytes;
while (to_last_block_bytes > 0 && iovcnt > 0) {
iov_len = spdk_min(to_last_block_bytes,
bdev_io->child_iov[child_iovpos].iov_len);
bdev_io->child_iov[child_iovpos].iov_len -= iov_len;
if (bdev_io->child_iov[child_iovpos].iov_len == 0) {
child_iovpos--;
if (--iovcnt == 0) {
/* If the child IO is less than a block size just return.
* If the first child IO of any split round is less than
* a block size, an error exit.
*/
if (bdev_io->u.bdev.split_outstanding == 0) {
SPDK_ERRLOG("The first child io was less than a block size\n");
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
spdk_trace_record(TRACE_BDEV_IO_DONE, 0, 0, (uintptr_t)bdev_io);
TAILQ_REMOVE(&bdev_io->internal.ch->io_submitted, bdev_io, internal.ch_link);
bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
}
return;
}
}
to_last_block_bytes -= iov_len;
if (parent_iov_offset == 0) {
parent_iovpos--;
parent_iov_offset = bdev_io->u.bdev.iovs[parent_iovpos].iov_len;
}
parent_iov_offset -= iov_len;
}
assert(to_last_block_bytes == 0);
}
to_next_boundary -= to_next_boundary_bytes / blocklen;
}
rc = bdev_io_split_submit(bdev_io, iov, iovcnt, md_buf, to_next_boundary,
&current_offset, &remaining);
if (spdk_unlikely(rc)) {
return;
}
}
}
static void
bdev_unmap_split(struct spdk_bdev_io *bdev_io)
{
uint64_t offset, unmap_blocks, remaining, max_unmap_blocks;
uint32_t num_children_reqs = 0;
int rc;
offset = bdev_io->u.bdev.split_current_offset_blocks;
remaining = bdev_io->u.bdev.split_remaining_num_blocks;
max_unmap_blocks = bdev_io->bdev->max_unmap * bdev_io->bdev->max_unmap_segments;
while (remaining && (num_children_reqs < SPDK_BDEV_MAX_CHILDREN_UNMAP_WRITE_ZEROES_REQS)) {
unmap_blocks = spdk_min(remaining, max_unmap_blocks);
rc = bdev_io_split_submit(bdev_io, NULL, 0, NULL, unmap_blocks,
&offset, &remaining);
if (spdk_likely(rc == 0)) {
num_children_reqs++;
} else {
return;
}
}
}
static void
bdev_write_zeroes_split(struct spdk_bdev_io *bdev_io)
{
uint64_t offset, write_zeroes_blocks, remaining;
uint32_t num_children_reqs = 0;
int rc;
offset = bdev_io->u.bdev.split_current_offset_blocks;
remaining = bdev_io->u.bdev.split_remaining_num_blocks;
while (remaining && (num_children_reqs < SPDK_BDEV_MAX_CHILDREN_UNMAP_WRITE_ZEROES_REQS)) {
write_zeroes_blocks = spdk_min(remaining, bdev_io->bdev->max_write_zeroes);
rc = bdev_io_split_submit(bdev_io, NULL, 0, NULL, write_zeroes_blocks,
&offset, &remaining);
if (spdk_likely(rc == 0)) {
num_children_reqs++;
} else {
return;
}
}
}
static void
bdev_io_split_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
{
struct spdk_bdev_io *parent_io = cb_arg;
spdk_bdev_free_io(bdev_io);
if (!success) {
parent_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
/* If any child I/O failed, stop further splitting process. */
parent_io->u.bdev.split_current_offset_blocks += parent_io->u.bdev.split_remaining_num_blocks;
parent_io->u.bdev.split_remaining_num_blocks = 0;
}
parent_io->u.bdev.split_outstanding--;
if (parent_io->u.bdev.split_outstanding != 0) {
return;
}
/*
* Parent I/O finishes when all blocks are consumed.
*/
if (parent_io->u.bdev.split_remaining_num_blocks == 0) {
assert(parent_io->internal.cb != bdev_io_split_done);
spdk_trace_record(TRACE_BDEV_IO_DONE, 0, 0, (uintptr_t)parent_io);
TAILQ_REMOVE(&parent_io->internal.ch->io_submitted, parent_io, internal.ch_link);
parent_io->internal.cb(parent_io, parent_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS,
parent_io->internal.caller_ctx);
return;
}
/*
* Continue with the splitting process. This function will complete the parent I/O if the
* splitting is done.
*/
switch (parent_io->type) {
case SPDK_BDEV_IO_TYPE_READ:
case SPDK_BDEV_IO_TYPE_WRITE:
_bdev_rw_split(parent_io);
break;
case SPDK_BDEV_IO_TYPE_UNMAP:
bdev_unmap_split(parent_io);
break;
case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
bdev_write_zeroes_split(parent_io);
break;
default:
assert(false);
break;
}
}
static void
bdev_rw_split_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bool success);
static void
bdev_io_split(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io)
{
bdev_io->u.bdev.split_current_offset_blocks = bdev_io->u.bdev.offset_blocks;
bdev_io->u.bdev.split_remaining_num_blocks = bdev_io->u.bdev.num_blocks;
bdev_io->u.bdev.split_outstanding = 0;
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
switch (bdev_io->type) {
case SPDK_BDEV_IO_TYPE_READ:
case SPDK_BDEV_IO_TYPE_WRITE:
if (_is_buf_allocated(bdev_io->u.bdev.iovs)) {
_bdev_rw_split(bdev_io);
} else {
assert(bdev_io->type == SPDK_BDEV_IO_TYPE_READ);
spdk_bdev_io_get_buf(bdev_io, bdev_rw_split_get_buf_cb,
bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen);
}
break;
case SPDK_BDEV_IO_TYPE_UNMAP:
bdev_unmap_split(bdev_io);
break;
case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
bdev_write_zeroes_split(bdev_io);
break;
default:
assert(false);
break;
}
}
static void
bdev_rw_split_get_buf_cb(struct spdk_io_channel *ch, struct spdk_bdev_io *bdev_io, bool success)
{
if (!success) {
spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
return;
}
_bdev_rw_split(bdev_io);
}
/* Explicitly mark this inline, since it's used as a function pointer and otherwise won't
* be inlined, at least on some compilers.
*/
static inline void
_bdev_io_submit(void *ctx)
{
struct spdk_bdev_io *bdev_io = ctx;
struct spdk_bdev *bdev = bdev_io->bdev;
struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch;
uint64_t tsc;
tsc = spdk_get_ticks();
bdev_io->internal.submit_tsc = tsc;
spdk_trace_record_tsc(tsc, TRACE_BDEV_IO_START, 0, 0, (uintptr_t)bdev_io, bdev_io->type);
if (spdk_likely(bdev_ch->flags == 0)) {
bdev_io_do_submit(bdev_ch, bdev_io);
return;
}
if (bdev_ch->flags & BDEV_CH_RESET_IN_PROGRESS) {
_bdev_io_complete_in_submit(bdev_ch, bdev_io, SPDK_BDEV_IO_STATUS_ABORTED);
} else if (bdev_ch->flags & BDEV_CH_QOS_ENABLED) {
if (spdk_unlikely(bdev_io->type == SPDK_BDEV_IO_TYPE_ABORT) &&
bdev_abort_queued_io(&bdev->internal.qos->queued, bdev_io->u.abort.bio_to_abort)) {
_bdev_io_complete_in_submit(bdev_ch, bdev_io, SPDK_BDEV_IO_STATUS_SUCCESS);
} else {
TAILQ_INSERT_TAIL(&bdev->internal.qos->queued, bdev_io, internal.link);
bdev_qos_io_submit(bdev_ch, bdev->internal.qos);
}
} else {
SPDK_ERRLOG("unknown bdev_ch flag %x found\n", bdev_ch->flags);
_bdev_io_complete_in_submit(bdev_ch, bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
}
}
bool
bdev_lba_range_overlapped(struct lba_range *range1, struct lba_range *range2);
bool
bdev_lba_range_overlapped(struct lba_range *range1, struct lba_range *range2)
{
if (range1->length == 0 || range2->length == 0) {
return false;
}
if (range1->offset + range1->length <= range2->offset) {
return false;
}
if (range2->offset + range2->length <= range1->offset) {
return false;
}
return true;
}
static bool
bdev_io_range_is_locked(struct spdk_bdev_io *bdev_io, struct lba_range *range)
{
struct spdk_bdev_channel *ch = bdev_io->internal.ch;
struct lba_range r;
switch (bdev_io->type) {
case SPDK_BDEV_IO_TYPE_NVME_IO:
case SPDK_BDEV_IO_TYPE_NVME_IO_MD:
/* Don't try to decode the NVMe command - just assume worst-case and that
* it overlaps a locked range.
*/
return true;
case SPDK_BDEV_IO_TYPE_WRITE:
case SPDK_BDEV_IO_TYPE_UNMAP:
case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
case SPDK_BDEV_IO_TYPE_ZCOPY:
r.offset = bdev_io->u.bdev.offset_blocks;
r.length = bdev_io->u.bdev.num_blocks;
if (!bdev_lba_range_overlapped(range, &r)) {
/* This I/O doesn't overlap the specified LBA range. */
return false;
} else if (range->owner_ch == ch && range->locked_ctx == bdev_io->internal.caller_ctx) {
/* This I/O overlaps, but the I/O is on the same channel that locked this
* range, and the caller_ctx is the same as the locked_ctx. This means
* that this I/O is associated with the lock, and is allowed to execute.
*/
return false;
} else {
return true;
}
default:
return false;
}
}
void
bdev_io_submit(struct spdk_bdev_io *bdev_io)
{
struct spdk_bdev *bdev = bdev_io->bdev;
struct spdk_thread *thread = spdk_bdev_io_get_thread(bdev_io);
struct spdk_bdev_channel *ch = bdev_io->internal.ch;
assert(thread != NULL);
assert(bdev_io->internal.status == SPDK_BDEV_IO_STATUS_PENDING);
if (!TAILQ_EMPTY(&ch->locked_ranges)) {
struct lba_range *range;
TAILQ_FOREACH(range, &ch->locked_ranges, tailq) {
if (bdev_io_range_is_locked(bdev_io, range)) {
TAILQ_INSERT_TAIL(&ch->io_locked, bdev_io, internal.ch_link);
return;
}
}
}
TAILQ_INSERT_TAIL(&ch->io_submitted, bdev_io, internal.ch_link);
if (bdev_io_should_split(bdev_io)) {
bdev_io->internal.submit_tsc = spdk_get_ticks();
spdk_trace_record_tsc(bdev_io->internal.submit_tsc, TRACE_BDEV_IO_START, 0, 0,
(uintptr_t)bdev_io, bdev_io->type);
bdev_io_split(NULL, bdev_io);
return;
}
if (ch->flags & BDEV_CH_QOS_ENABLED) {
if ((thread == bdev->internal.qos->thread) || !bdev->internal.qos->thread) {
_bdev_io_submit(bdev_io);
} else {
bdev_io->internal.io_submit_ch = ch;
bdev_io->internal.ch = bdev->internal.qos->ch;
spdk_thread_send_msg(bdev->internal.qos->thread, _bdev_io_submit, bdev_io);
}
} else {
_bdev_io_submit(bdev_io);
}
}
static void
bdev_io_submit_reset(struct spdk_bdev_io *bdev_io)
{
struct spdk_bdev *bdev = bdev_io->bdev;
struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch;
struct spdk_io_channel *ch = bdev_ch->channel;
assert(bdev_io->internal.status == SPDK_BDEV_IO_STATUS_PENDING);
bdev_io->internal.in_submit_request = true;
bdev->fn_table->submit_request(ch, bdev_io);
bdev_io->internal.in_submit_request = false;
}
void
bdev_io_init(struct spdk_bdev_io *bdev_io,
struct spdk_bdev *bdev, void *cb_arg,
spdk_bdev_io_completion_cb cb)
{
bdev_io->bdev = bdev;
bdev_io->internal.caller_ctx = cb_arg;
bdev_io->internal.cb = cb;
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_PENDING;
bdev_io->internal.in_submit_request = false;
bdev_io->internal.buf = NULL;
bdev_io->internal.io_submit_ch = NULL;
bdev_io->internal.orig_iovs = NULL;
bdev_io->internal.orig_iovcnt = 0;
bdev_io->internal.orig_md_buf = NULL;
bdev_io->internal.error.nvme.cdw0 = 0;
bdev_io->num_retries = 0;
bdev_io->internal.get_buf_cb = NULL;
bdev_io->internal.get_aux_buf_cb = NULL;
}
static bool
bdev_io_type_supported(struct spdk_bdev *bdev, enum spdk_bdev_io_type io_type)
{
return bdev->fn_table->io_type_supported(bdev->ctxt, io_type);
}
bool
spdk_bdev_io_type_supported(struct spdk_bdev *bdev, enum spdk_bdev_io_type io_type)
{
bool supported;
supported = bdev_io_type_supported(bdev, io_type);
if (!supported) {
switch (io_type) {
case SPDK_BDEV_IO_TYPE_WRITE_ZEROES:
/* The bdev layer will emulate write zeroes as long as write is supported. */
supported = bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_WRITE);
break;
default:
break;
}
}
return supported;
}
int
spdk_bdev_dump_info_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
{
if (bdev->fn_table->dump_info_json) {
return bdev->fn_table->dump_info_json(bdev->ctxt, w);
}
return 0;
}
static void
bdev_qos_update_max_quota_per_timeslice(struct spdk_bdev_qos *qos)
{
uint32_t max_per_timeslice = 0;
int i;
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
if (qos->rate_limits[i].limit == SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {
qos->rate_limits[i].max_per_timeslice = 0;
continue;
}
max_per_timeslice = qos->rate_limits[i].limit *
SPDK_BDEV_QOS_TIMESLICE_IN_USEC / SPDK_SEC_TO_USEC;
qos->rate_limits[i].max_per_timeslice = spdk_max(max_per_timeslice,
qos->rate_limits[i].min_per_timeslice);
qos->rate_limits[i].remaining_this_timeslice = qos->rate_limits[i].max_per_timeslice;
}
bdev_qos_set_ops(qos);
}
static int
bdev_channel_poll_qos(void *arg)
{
struct spdk_bdev_qos *qos = arg;
uint64_t now = spdk_get_ticks();
int i;
if (now < (qos->last_timeslice + qos->timeslice_size)) {
/* We received our callback earlier than expected - return
* immediately and wait to do accounting until at least one
* timeslice has actually expired. This should never happen
* with a well-behaved timer implementation.
*/
return SPDK_POLLER_IDLE;
}
/* Reset for next round of rate limiting */
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
/* We may have allowed the IOs or bytes to slightly overrun in the last
* timeslice. remaining_this_timeslice is signed, so if it's negative
* here, we'll account for the overrun so that the next timeslice will
* be appropriately reduced.
*/
if (qos->rate_limits[i].remaining_this_timeslice > 0) {
qos->rate_limits[i].remaining_this_timeslice = 0;
}
}
while (now >= (qos->last_timeslice + qos->timeslice_size)) {
qos->last_timeslice += qos->timeslice_size;
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
qos->rate_limits[i].remaining_this_timeslice +=
qos->rate_limits[i].max_per_timeslice;
}
}
return bdev_qos_io_submit(qos->ch, qos);
}
static void
bdev_channel_destroy_resource(struct spdk_bdev_channel *ch)
{
struct spdk_bdev_shared_resource *shared_resource;
struct lba_range *range;
while (!TAILQ_EMPTY(&ch->locked_ranges)) {
range = TAILQ_FIRST(&ch->locked_ranges);
TAILQ_REMOVE(&ch->locked_ranges, range, tailq);
free(range);
}
spdk_put_io_channel(ch->channel);
shared_resource = ch->shared_resource;
assert(TAILQ_EMPTY(&ch->io_locked));
assert(TAILQ_EMPTY(&ch->io_submitted));
assert(ch->io_outstanding == 0);
assert(shared_resource->ref > 0);
shared_resource->ref--;
if (shared_resource->ref == 0) {
assert(shared_resource->io_outstanding == 0);
TAILQ_REMOVE(&shared_resource->mgmt_ch->shared_resources, shared_resource, link);
spdk_put_io_channel(spdk_io_channel_from_ctx(shared_resource->mgmt_ch));
free(shared_resource);
}
}
/* Caller must hold bdev->internal.mutex. */
static void
bdev_enable_qos(struct spdk_bdev *bdev, struct spdk_bdev_channel *ch)
{
struct spdk_bdev_qos *qos = bdev->internal.qos;
int i;
/* Rate limiting on this bdev enabled */
if (qos) {
if (qos->ch == NULL) {
struct spdk_io_channel *io_ch;
SPDK_DEBUGLOG(bdev, "Selecting channel %p as QoS channel for bdev %s on thread %p\n", ch,
bdev->name, spdk_get_thread());
/* No qos channel has been selected, so set one up */
/* Take another reference to ch */
io_ch = spdk_get_io_channel(__bdev_to_io_dev(bdev));
assert(io_ch != NULL);
qos->ch = ch;
qos->thread = spdk_io_channel_get_thread(io_ch);
TAILQ_INIT(&qos->queued);
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
if (bdev_qos_is_iops_rate_limit(i) == true) {
qos->rate_limits[i].min_per_timeslice =
SPDK_BDEV_QOS_MIN_IO_PER_TIMESLICE;
} else {
qos->rate_limits[i].min_per_timeslice =
SPDK_BDEV_QOS_MIN_BYTE_PER_TIMESLICE;
}
if (qos->rate_limits[i].limit == 0) {
qos->rate_limits[i].limit = SPDK_BDEV_QOS_LIMIT_NOT_DEFINED;
}
}
bdev_qos_update_max_quota_per_timeslice(qos);
qos->timeslice_size =
SPDK_BDEV_QOS_TIMESLICE_IN_USEC * spdk_get_ticks_hz() / SPDK_SEC_TO_USEC;
qos->last_timeslice = spdk_get_ticks();
qos->poller = SPDK_POLLER_REGISTER(bdev_channel_poll_qos,
qos,
SPDK_BDEV_QOS_TIMESLICE_IN_USEC);
}
ch->flags |= BDEV_CH_QOS_ENABLED;
}
}
struct poll_timeout_ctx {
struct spdk_bdev_desc *desc;
uint64_t timeout_in_sec;
spdk_bdev_io_timeout_cb cb_fn;
void *cb_arg;
};
static void
bdev_desc_free(struct spdk_bdev_desc *desc)
{
pthread_mutex_destroy(&desc->mutex);
free(desc->media_events_buffer);
free(desc);
}
static void
bdev_channel_poll_timeout_io_done(struct spdk_io_channel_iter *i, int status)
{
struct poll_timeout_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
struct spdk_bdev_desc *desc = ctx->desc;
free(ctx);
pthread_mutex_lock(&desc->mutex);
desc->refs--;
if (desc->closed == true && desc->refs == 0) {
pthread_mutex_unlock(&desc->mutex);
bdev_desc_free(desc);
return;
}
pthread_mutex_unlock(&desc->mutex);
}
static void
bdev_channel_poll_timeout_io(struct spdk_io_channel_iter *i)
{
struct poll_timeout_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
struct spdk_io_channel *io_ch = spdk_io_channel_iter_get_channel(i);
struct spdk_bdev_channel *bdev_ch = spdk_io_channel_get_ctx(io_ch);
struct spdk_bdev_desc *desc = ctx->desc;
struct spdk_bdev_io *bdev_io;
uint64_t now;
pthread_mutex_lock(&desc->mutex);
if (desc->closed == true) {
pthread_mutex_unlock(&desc->mutex);
spdk_for_each_channel_continue(i, -1);
return;
}
pthread_mutex_unlock(&desc->mutex);
now = spdk_get_ticks();
TAILQ_FOREACH(bdev_io, &bdev_ch->io_submitted, internal.ch_link) {
/* Exclude any I/O that are generated via splitting. */
if (bdev_io->internal.cb == bdev_io_split_done) {
continue;
}
/* Once we find an I/O that has not timed out, we can immediately
* exit the loop.
*/
if (now < (bdev_io->internal.submit_tsc +
ctx->timeout_in_sec * spdk_get_ticks_hz())) {
goto end;
}
if (bdev_io->internal.desc == desc) {
ctx->cb_fn(ctx->cb_arg, bdev_io);
}
}
end:
spdk_for_each_channel_continue(i, 0);
}
static int
bdev_poll_timeout_io(void *arg)
{
struct spdk_bdev_desc *desc = arg;
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct poll_timeout_ctx *ctx;
ctx = calloc(1, sizeof(struct poll_timeout_ctx));
if (!ctx) {
SPDK_ERRLOG("failed to allocate memory\n");
return SPDK_POLLER_BUSY;
}
ctx->desc = desc;
ctx->cb_arg = desc->cb_arg;
ctx->cb_fn = desc->cb_fn;
ctx->timeout_in_sec = desc->timeout_in_sec;
/* Take a ref on the descriptor in case it gets closed while we are checking
* all of the channels.
*/
pthread_mutex_lock(&desc->mutex);
desc->refs++;
pthread_mutex_unlock(&desc->mutex);
spdk_for_each_channel(__bdev_to_io_dev(bdev),
bdev_channel_poll_timeout_io,
ctx,
bdev_channel_poll_timeout_io_done);
return SPDK_POLLER_BUSY;
}
int
spdk_bdev_set_timeout(struct spdk_bdev_desc *desc, uint64_t timeout_in_sec,
spdk_bdev_io_timeout_cb cb_fn, void *cb_arg)
{
assert(desc->thread == spdk_get_thread());
spdk_poller_unregister(&desc->io_timeout_poller);
if (timeout_in_sec) {
assert(cb_fn != NULL);
desc->io_timeout_poller = SPDK_POLLER_REGISTER(bdev_poll_timeout_io,
desc,
SPDK_BDEV_IO_POLL_INTERVAL_IN_MSEC * SPDK_SEC_TO_USEC /
1000);
if (desc->io_timeout_poller == NULL) {
SPDK_ERRLOG("can not register the desc timeout IO poller\n");
return -1;
}
}
desc->cb_fn = cb_fn;
desc->cb_arg = cb_arg;
desc->timeout_in_sec = timeout_in_sec;
return 0;
}
static int
bdev_channel_create(void *io_device, void *ctx_buf)
{
struct spdk_bdev *bdev = __bdev_from_io_dev(io_device);
struct spdk_bdev_channel *ch = ctx_buf;
struct spdk_io_channel *mgmt_io_ch;
struct spdk_bdev_mgmt_channel *mgmt_ch;
struct spdk_bdev_shared_resource *shared_resource;
struct lba_range *range;
ch->bdev = bdev;
ch->channel = bdev->fn_table->get_io_channel(bdev->ctxt);
if (!ch->channel) {
return -1;
}
assert(ch->histogram == NULL);
if (bdev->internal.histogram_enabled) {
ch->histogram = spdk_histogram_data_alloc();
if (ch->histogram == NULL) {
SPDK_ERRLOG("Could not allocate histogram\n");
}
}
mgmt_io_ch = spdk_get_io_channel(&g_bdev_mgr);
if (!mgmt_io_ch) {
spdk_put_io_channel(ch->channel);
return -1;
}
mgmt_ch = spdk_io_channel_get_ctx(mgmt_io_ch);
TAILQ_FOREACH(shared_resource, &mgmt_ch->shared_resources, link) {
if (shared_resource->shared_ch == ch->channel) {
spdk_put_io_channel(mgmt_io_ch);
shared_resource->ref++;
break;
}
}
if (shared_resource == NULL) {
shared_resource = calloc(1, sizeof(*shared_resource));
if (shared_resource == NULL) {
spdk_put_io_channel(ch->channel);
spdk_put_io_channel(mgmt_io_ch);
return -1;
}
shared_resource->mgmt_ch = mgmt_ch;
shared_resource->io_outstanding = 0;
TAILQ_INIT(&shared_resource->nomem_io);
shared_resource->nomem_threshold = 0;
shared_resource->shared_ch = ch->channel;
shared_resource->ref = 1;
TAILQ_INSERT_TAIL(&mgmt_ch->shared_resources, shared_resource, link);
}
memset(&ch->stat, 0, sizeof(ch->stat));
ch->stat.ticks_rate = spdk_get_ticks_hz();
ch->io_outstanding = 0;
TAILQ_INIT(&ch->queued_resets);
TAILQ_INIT(&ch->locked_ranges);
ch->flags = 0;
ch->shared_resource = shared_resource;
TAILQ_INIT(&ch->io_submitted);
TAILQ_INIT(&ch->io_locked);
#ifdef SPDK_CONFIG_VTUNE
{
char *name;
__itt_init_ittlib(NULL, 0);
name = spdk_sprintf_alloc("spdk_bdev_%s_%p", ch->bdev->name, ch);
if (!name) {
bdev_channel_destroy_resource(ch);
return -1;
}
ch->handle = __itt_string_handle_create(name);
free(name);
ch->start_tsc = spdk_get_ticks();
ch->interval_tsc = spdk_get_ticks_hz() / 100;
memset(&ch->prev_stat, 0, sizeof(ch->prev_stat));
}
#endif
pthread_mutex_lock(&bdev->internal.mutex);
bdev_enable_qos(bdev, ch);
TAILQ_FOREACH(range, &bdev->internal.locked_ranges, tailq) {
struct lba_range *new_range;
new_range = calloc(1, sizeof(*new_range));
if (new_range == NULL) {
pthread_mutex_unlock(&bdev->internal.mutex);
bdev_channel_destroy_resource(ch);
return -1;
}
new_range->length = range->length;
new_range->offset = range->offset;
new_range->locked_ctx = range->locked_ctx;
TAILQ_INSERT_TAIL(&ch->locked_ranges, new_range, tailq);
}
pthread_mutex_unlock(&bdev->internal.mutex);
return 0;
}
/*
* Abort I/O that are waiting on a data buffer. These types of I/O are
* linked using the spdk_bdev_io internal.buf_link TAILQ_ENTRY.
*/
static void
bdev_abort_all_buf_io(bdev_io_stailq_t *queue, struct spdk_bdev_channel *ch)
{
bdev_io_stailq_t tmp;
struct spdk_bdev_io *bdev_io;
STAILQ_INIT(&tmp);
while (!STAILQ_EMPTY(queue)) {
bdev_io = STAILQ_FIRST(queue);
STAILQ_REMOVE_HEAD(queue, internal.buf_link);
if (bdev_io->internal.ch == ch) {
spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_ABORTED);
} else {
STAILQ_INSERT_TAIL(&tmp, bdev_io, internal.buf_link);
}
}
STAILQ_SWAP(&tmp, queue, spdk_bdev_io);
}
/*
* Abort I/O that are queued waiting for submission. These types of I/O are
* linked using the spdk_bdev_io link TAILQ_ENTRY.
*/
static void
bdev_abort_all_queued_io(bdev_io_tailq_t *queue, struct spdk_bdev_channel *ch)
{
struct spdk_bdev_io *bdev_io, *tmp;
TAILQ_FOREACH_SAFE(bdev_io, queue, internal.link, tmp) {
if (bdev_io->internal.ch == ch) {
TAILQ_REMOVE(queue, bdev_io, internal.link);
/*
* spdk_bdev_io_complete() assumes that the completed I/O had
* been submitted to the bdev module. Since in this case it
* hadn't, bump io_outstanding to account for the decrement
* that spdk_bdev_io_complete() will do.
*/
if (bdev_io->type != SPDK_BDEV_IO_TYPE_RESET) {
ch->io_outstanding++;
ch->shared_resource->io_outstanding++;
}
spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_ABORTED);
}
}
}
static bool
bdev_abort_queued_io(bdev_io_tailq_t *queue, struct spdk_bdev_io *bio_to_abort)
{
struct spdk_bdev_io *bdev_io;
TAILQ_FOREACH(bdev_io, queue, internal.link) {
if (bdev_io == bio_to_abort) {
TAILQ_REMOVE(queue, bio_to_abort, internal.link);
spdk_bdev_io_complete(bio_to_abort, SPDK_BDEV_IO_STATUS_ABORTED);
return true;
}
}
return false;
}
static bool
bdev_abort_buf_io(bdev_io_stailq_t *queue, struct spdk_bdev_io *bio_to_abort)
{
struct spdk_bdev_io *bdev_io;
STAILQ_FOREACH(bdev_io, queue, internal.buf_link) {
if (bdev_io == bio_to_abort) {
STAILQ_REMOVE(queue, bio_to_abort, spdk_bdev_io, internal.buf_link);
spdk_bdev_io_complete(bio_to_abort, SPDK_BDEV_IO_STATUS_ABORTED);
return true;
}
}
return false;
}
static void
bdev_qos_channel_destroy(void *cb_arg)
{
struct spdk_bdev_qos *qos = cb_arg;
spdk_put_io_channel(spdk_io_channel_from_ctx(qos->ch));
spdk_poller_unregister(&qos->poller);
SPDK_DEBUGLOG(bdev, "Free QoS %p.\n", qos);
free(qos);
}
static int
bdev_qos_destroy(struct spdk_bdev *bdev)
{
int i;
/*
* Cleanly shutting down the QoS poller is tricky, because
* during the asynchronous operation the user could open
* a new descriptor and create a new channel, spawning
* a new QoS poller.
*
* The strategy is to create a new QoS structure here and swap it
* in. The shutdown path then continues to refer to the old one
* until it completes and then releases it.
*/
struct spdk_bdev_qos *new_qos, *old_qos;
old_qos = bdev->internal.qos;
new_qos = calloc(1, sizeof(*new_qos));
if (!new_qos) {
SPDK_ERRLOG("Unable to allocate memory to shut down QoS.\n");
return -ENOMEM;
}
/* Copy the old QoS data into the newly allocated structure */
memcpy(new_qos, old_qos, sizeof(*new_qos));
/* Zero out the key parts of the QoS structure */
new_qos->ch = NULL;
new_qos->thread = NULL;
new_qos->poller = NULL;
TAILQ_INIT(&new_qos->queued);
/*
* The limit member of spdk_bdev_qos_limit structure is not zeroed.
* It will be used later for the new QoS structure.
*/
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
new_qos->rate_limits[i].remaining_this_timeslice = 0;
new_qos->rate_limits[i].min_per_timeslice = 0;
new_qos->rate_limits[i].max_per_timeslice = 0;
}
bdev->internal.qos = new_qos;
if (old_qos->thread == NULL) {
free(old_qos);
} else {
spdk_thread_send_msg(old_qos->thread, bdev_qos_channel_destroy, old_qos);
}
/* It is safe to continue with destroying the bdev even though the QoS channel hasn't
* been destroyed yet. The destruction path will end up waiting for the final
* channel to be put before it releases resources. */
return 0;
}
static void
bdev_io_stat_add(struct spdk_bdev_io_stat *total, struct spdk_bdev_io_stat *add)
{
total->bytes_read += add->bytes_read;
total->num_read_ops += add->num_read_ops;
total->bytes_written += add->bytes_written;
total->num_write_ops += add->num_write_ops;
total->bytes_unmapped += add->bytes_unmapped;
total->num_unmap_ops += add->num_unmap_ops;
total->read_latency_ticks += add->read_latency_ticks;
total->write_latency_ticks += add->write_latency_ticks;
total->unmap_latency_ticks += add->unmap_latency_ticks;
}
static void
bdev_channel_destroy(void *io_device, void *ctx_buf)
{
struct spdk_bdev_channel *ch = ctx_buf;
struct spdk_bdev_mgmt_channel *mgmt_ch;
struct spdk_bdev_shared_resource *shared_resource = ch->shared_resource;
SPDK_DEBUGLOG(bdev, "Destroying channel %p for bdev %s on thread %p\n", ch, ch->bdev->name,
spdk_get_thread());
/* This channel is going away, so add its statistics into the bdev so that they don't get lost. */
pthread_mutex_lock(&ch->bdev->internal.mutex);
bdev_io_stat_add(&ch->bdev->internal.stat, &ch->stat);
pthread_mutex_unlock(&ch->bdev->internal.mutex);
mgmt_ch = shared_resource->mgmt_ch;
bdev_abort_all_queued_io(&ch->queued_resets, ch);
bdev_abort_all_queued_io(&shared_resource->nomem_io, ch);
bdev_abort_all_buf_io(&mgmt_ch->need_buf_small, ch);
bdev_abort_all_buf_io(&mgmt_ch->need_buf_large, ch);
if (ch->histogram) {
spdk_histogram_data_free(ch->histogram);
}
bdev_channel_destroy_resource(ch);
}
/*
* If the name already exists in the global bdev name tree, RB_INSERT() returns a pointer
* to it. Hence we do not have to call bdev_get_by_name() when using this function.
*/
static int
bdev_name_add(struct spdk_bdev_name *bdev_name, struct spdk_bdev *bdev, const char *name)
{
struct spdk_bdev_name *tmp;
bdev_name->name = strdup(name);
if (bdev_name->name == NULL) {
SPDK_ERRLOG("Unable to allocate bdev name\n");
return -ENOMEM;
}
bdev_name->bdev = bdev;
pthread_mutex_lock(&g_bdev_mgr.mutex);
tmp = RB_INSERT(bdev_name_tree, &g_bdev_mgr.bdev_names, bdev_name);
pthread_mutex_unlock(&g_bdev_mgr.mutex);
if (tmp != NULL) {
SPDK_ERRLOG("Bdev name %s already exists\n", name);
free(bdev_name->name);
return -EEXIST;
}
return 0;
}
static void
bdev_name_del(struct spdk_bdev_name *bdev_name)
{
RB_REMOVE(bdev_name_tree, &g_bdev_mgr.bdev_names, bdev_name);
free(bdev_name->name);
}
int
spdk_bdev_alias_add(struct spdk_bdev *bdev, const char *alias)
{
struct spdk_bdev_alias *tmp;
int ret;
if (alias == NULL) {
SPDK_ERRLOG("Empty alias passed\n");
return -EINVAL;
}
tmp = calloc(1, sizeof(*tmp));
if (tmp == NULL) {
SPDK_ERRLOG("Unable to allocate alias\n");
return -ENOMEM;
}
ret = bdev_name_add(&tmp->alias, bdev, alias);
if (ret != 0) {
free(tmp);
return ret;
}
TAILQ_INSERT_TAIL(&bdev->aliases, tmp, tailq);
return 0;
}
int
spdk_bdev_alias_del(struct spdk_bdev *bdev, const char *alias)
{
struct spdk_bdev_alias *tmp;
TAILQ_FOREACH(tmp, &bdev->aliases, tailq) {
if (strcmp(alias, tmp->alias.name) == 0) {
TAILQ_REMOVE(&bdev->aliases, tmp, tailq);
pthread_mutex_lock(&g_bdev_mgr.mutex);
bdev_name_del(&tmp->alias);
pthread_mutex_unlock(&g_bdev_mgr.mutex);
free(tmp);
return 0;
}
}
SPDK_INFOLOG(bdev, "Alias %s does not exists\n", alias);
return -ENOENT;
}
void
spdk_bdev_alias_del_all(struct spdk_bdev *bdev)
{
struct spdk_bdev_alias *p, *tmp;
TAILQ_FOREACH_SAFE(p, &bdev->aliases, tailq, tmp) {
TAILQ_REMOVE(&bdev->aliases, p, tailq);
pthread_mutex_lock(&g_bdev_mgr.mutex);
bdev_name_del(&p->alias);
pthread_mutex_unlock(&g_bdev_mgr.mutex);
free(p);
}
}
struct spdk_io_channel *
spdk_bdev_get_io_channel(struct spdk_bdev_desc *desc)
{
return spdk_get_io_channel(__bdev_to_io_dev(spdk_bdev_desc_get_bdev(desc)));
}
void *
spdk_bdev_get_module_ctx(struct spdk_bdev_desc *desc)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
void *ctx = NULL;
if (bdev->fn_table->get_module_ctx) {
ctx = bdev->fn_table->get_module_ctx(bdev->ctxt);
}
return ctx;
}
const char *
spdk_bdev_get_module_name(const struct spdk_bdev *bdev)
{
return bdev->module->name;
}
const char *
spdk_bdev_get_name(const struct spdk_bdev *bdev)
{
return bdev->name;
}
const char *
spdk_bdev_get_product_name(const struct spdk_bdev *bdev)
{
return bdev->product_name;
}
const struct spdk_bdev_aliases_list *
spdk_bdev_get_aliases(const struct spdk_bdev *bdev)
{
return &bdev->aliases;
}
uint32_t
spdk_bdev_get_block_size(const struct spdk_bdev *bdev)
{
return bdev->blocklen;
}
uint32_t
spdk_bdev_get_write_unit_size(const struct spdk_bdev *bdev)
{
return bdev->write_unit_size;
}
uint64_t
spdk_bdev_get_num_blocks(const struct spdk_bdev *bdev)
{
return bdev->blockcnt;
}
const char *
spdk_bdev_get_qos_rpc_type(enum spdk_bdev_qos_rate_limit_type type)
{
return qos_rpc_type[type];
}
void
spdk_bdev_get_qos_rate_limits(struct spdk_bdev *bdev, uint64_t *limits)
{
int i;
memset(limits, 0, sizeof(*limits) * SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES);
pthread_mutex_lock(&bdev->internal.mutex);
if (bdev->internal.qos) {
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
if (bdev->internal.qos->rate_limits[i].limit !=
SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {
limits[i] = bdev->internal.qos->rate_limits[i].limit;
if (bdev_qos_is_iops_rate_limit(i) == false) {
/* Change from Byte to Megabyte which is user visible. */
limits[i] = limits[i] / 1024 / 1024;
}
}
}
}
pthread_mutex_unlock(&bdev->internal.mutex);
}
size_t
spdk_bdev_get_buf_align(const struct spdk_bdev *bdev)
{
return 1 << bdev->required_alignment;
}
uint32_t
spdk_bdev_get_optimal_io_boundary(const struct spdk_bdev *bdev)
{
return bdev->optimal_io_boundary;
}
bool
spdk_bdev_has_write_cache(const struct spdk_bdev *bdev)
{
return bdev->write_cache;
}
const struct spdk_uuid *
spdk_bdev_get_uuid(const struct spdk_bdev *bdev)
{
return &bdev->uuid;
}
uint16_t
spdk_bdev_get_acwu(const struct spdk_bdev *bdev)
{
return bdev->acwu;
}
uint32_t
spdk_bdev_get_md_size(const struct spdk_bdev *bdev)
{
return bdev->md_len;
}
bool
spdk_bdev_is_md_interleaved(const struct spdk_bdev *bdev)
{
return (bdev->md_len != 0) && bdev->md_interleave;
}
bool
spdk_bdev_is_md_separate(const struct spdk_bdev *bdev)
{
return (bdev->md_len != 0) && !bdev->md_interleave;
}
bool
spdk_bdev_is_zoned(const struct spdk_bdev *bdev)
{
return bdev->zoned;
}
uint32_t
spdk_bdev_get_data_block_size(const struct spdk_bdev *bdev)
{
if (spdk_bdev_is_md_interleaved(bdev)) {
return bdev->blocklen - bdev->md_len;
} else {
return bdev->blocklen;
}
}
uint32_t
spdk_bdev_get_physical_block_size(const struct spdk_bdev *bdev)
{
return bdev->phys_blocklen;
}
static uint32_t
_bdev_get_block_size_with_md(const struct spdk_bdev *bdev)
{
if (!spdk_bdev_is_md_interleaved(bdev)) {
return bdev->blocklen + bdev->md_len;
} else {
return bdev->blocklen;
}
}
enum spdk_dif_type spdk_bdev_get_dif_type(const struct spdk_bdev *bdev)
{
if (bdev->md_len != 0) {
return bdev->dif_type;
} else {
return SPDK_DIF_DISABLE;
}
}
bool
spdk_bdev_is_dif_head_of_md(const struct spdk_bdev *bdev)
{
if (spdk_bdev_get_dif_type(bdev) != SPDK_DIF_DISABLE) {
return bdev->dif_is_head_of_md;
} else {
return false;
}
}
bool
spdk_bdev_is_dif_check_enabled(const struct spdk_bdev *bdev,
enum spdk_dif_check_type check_type)
{
if (spdk_bdev_get_dif_type(bdev) == SPDK_DIF_DISABLE) {
return false;
}
switch (check_type) {
case SPDK_DIF_CHECK_TYPE_REFTAG:
return (bdev->dif_check_flags & SPDK_DIF_FLAGS_REFTAG_CHECK) != 0;
case SPDK_DIF_CHECK_TYPE_APPTAG:
return (bdev->dif_check_flags & SPDK_DIF_FLAGS_APPTAG_CHECK) != 0;
case SPDK_DIF_CHECK_TYPE_GUARD:
return (bdev->dif_check_flags & SPDK_DIF_FLAGS_GUARD_CHECK) != 0;
default:
return false;
}
}
uint64_t
spdk_bdev_get_qd(const struct spdk_bdev *bdev)
{
return bdev->internal.measured_queue_depth;
}
uint64_t
spdk_bdev_get_qd_sampling_period(const struct spdk_bdev *bdev)
{
return bdev->internal.period;
}
uint64_t
spdk_bdev_get_weighted_io_time(const struct spdk_bdev *bdev)
{
return bdev->internal.weighted_io_time;
}
uint64_t
spdk_bdev_get_io_time(const struct spdk_bdev *bdev)
{
return bdev->internal.io_time;
}
static void
_calculate_measured_qd_cpl(struct spdk_io_channel_iter *i, int status)
{
struct spdk_bdev *bdev = spdk_io_channel_iter_get_ctx(i);
bdev->internal.measured_queue_depth = bdev->internal.temporary_queue_depth;
if (bdev->internal.measured_queue_depth) {
bdev->internal.io_time += bdev->internal.period;
bdev->internal.weighted_io_time += bdev->internal.period * bdev->internal.measured_queue_depth;
}
}
static void
_calculate_measured_qd(struct spdk_io_channel_iter *i)
{
struct spdk_bdev *bdev = spdk_io_channel_iter_get_ctx(i);
struct spdk_io_channel *io_ch = spdk_io_channel_iter_get_channel(i);
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(io_ch);
bdev->internal.temporary_queue_depth += ch->io_outstanding;
spdk_for_each_channel_continue(i, 0);
}
static int
bdev_calculate_measured_queue_depth(void *ctx)
{
struct spdk_bdev *bdev = ctx;
bdev->internal.temporary_queue_depth = 0;
spdk_for_each_channel(__bdev_to_io_dev(bdev), _calculate_measured_qd, bdev,
_calculate_measured_qd_cpl);
return SPDK_POLLER_BUSY;
}
void
spdk_bdev_set_qd_sampling_period(struct spdk_bdev *bdev, uint64_t period)
{
bdev->internal.period = period;
if (bdev->internal.qd_poller != NULL) {
spdk_poller_unregister(&bdev->internal.qd_poller);
bdev->internal.measured_queue_depth = UINT64_MAX;
}
if (period != 0) {
bdev->internal.qd_poller = SPDK_POLLER_REGISTER(bdev_calculate_measured_queue_depth, bdev,
period);
}
}
static void
_resize_notify(void *arg)
{
struct spdk_bdev_desc *desc = arg;
pthread_mutex_lock(&desc->mutex);
desc->refs--;
if (!desc->closed) {
pthread_mutex_unlock(&desc->mutex);
desc->callback.event_fn(SPDK_BDEV_EVENT_RESIZE,
desc->bdev,
desc->callback.ctx);
return;
} else if (0 == desc->refs) {
/* This descriptor was closed after this resize_notify message was sent.
* spdk_bdev_close() could not free the descriptor since this message was
* in flight, so we free it now using bdev_desc_free().
*/
pthread_mutex_unlock(&desc->mutex);
bdev_desc_free(desc);
return;
}
pthread_mutex_unlock(&desc->mutex);
}
int
spdk_bdev_notify_blockcnt_change(struct spdk_bdev *bdev, uint64_t size)
{
struct spdk_bdev_desc *desc;
int ret;
if (size == bdev->blockcnt) {
return 0;
}
pthread_mutex_lock(&bdev->internal.mutex);
/* bdev has open descriptors */
if (!TAILQ_EMPTY(&bdev->internal.open_descs) &&
bdev->blockcnt > size) {
ret = -EBUSY;
} else {
bdev->blockcnt = size;
TAILQ_FOREACH(desc, &bdev->internal.open_descs, link) {
pthread_mutex_lock(&desc->mutex);
if (!desc->closed) {
desc->refs++;
spdk_thread_send_msg(desc->thread, _resize_notify, desc);
}
pthread_mutex_unlock(&desc->mutex);
}
ret = 0;
}
pthread_mutex_unlock(&bdev->internal.mutex);
return ret;
}
/*
* Convert I/O offset and length from bytes to blocks.
*
* Returns zero on success or non-zero if the byte parameters aren't divisible by the block size.
*/
static uint64_t
bdev_bytes_to_blocks(struct spdk_bdev *bdev, uint64_t offset_bytes, uint64_t *offset_blocks,
uint64_t num_bytes, uint64_t *num_blocks)
{
uint32_t block_size = bdev->blocklen;
uint8_t shift_cnt;
/* Avoid expensive div operations if possible. These spdk_u32 functions are very cheap. */
if (spdk_likely(spdk_u32_is_pow2(block_size))) {
shift_cnt = spdk_u32log2(block_size);
*offset_blocks = offset_bytes >> shift_cnt;
*num_blocks = num_bytes >> shift_cnt;
return (offset_bytes - (*offset_blocks << shift_cnt)) |
(num_bytes - (*num_blocks << shift_cnt));
} else {
*offset_blocks = offset_bytes / block_size;
*num_blocks = num_bytes / block_size;
return (offset_bytes % block_size) | (num_bytes % block_size);
}
}
static bool
bdev_io_valid_blocks(struct spdk_bdev *bdev, uint64_t offset_blocks, uint64_t num_blocks)
{
/* Return failure if offset_blocks + num_blocks is less than offset_blocks; indicates there
* has been an overflow and hence the offset has been wrapped around */
if (offset_blocks + num_blocks < offset_blocks) {
return false;
}
/* Return failure if offset_blocks + num_blocks exceeds the size of the bdev */
if (offset_blocks + num_blocks > bdev->blockcnt) {
return false;
}
return true;
}
static bool
_bdev_io_check_md_buf(const struct iovec *iovs, const void *md_buf)
{
return _is_buf_allocated(iovs) == (md_buf != NULL);
}
static int
bdev_read_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch, void *buf,
void *md_buf, uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
return -EINVAL;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_READ;
bdev_io->u.bdev.iovs = &bdev_io->iov;
bdev_io->u.bdev.iovs[0].iov_base = buf;
bdev_io->u.bdev.iovs[0].iov_len = num_blocks * bdev->blocklen;
bdev_io->u.bdev.iovcnt = 1;
bdev_io->u.bdev.md_buf = md_buf;
bdev_io->u.bdev.num_blocks = num_blocks;
bdev_io->u.bdev.offset_blocks = offset_blocks;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
bdev_io_submit(bdev_io);
return 0;
}
int
spdk_bdev_read(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
void *buf, uint64_t offset, uint64_t nbytes,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
uint64_t offset_blocks, num_blocks;
if (bdev_bytes_to_blocks(spdk_bdev_desc_get_bdev(desc), offset, &offset_blocks,
nbytes, &num_blocks) != 0) {
return -EINVAL;
}
return spdk_bdev_read_blocks(desc, ch, buf, offset_blocks, num_blocks, cb, cb_arg);
}
int
spdk_bdev_read_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
void *buf, uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
return bdev_read_blocks_with_md(desc, ch, buf, NULL, offset_blocks, num_blocks, cb, cb_arg);
}
int
spdk_bdev_read_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
void *buf, void *md_buf, uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct iovec iov = {
.iov_base = buf,
};
if (!spdk_bdev_is_md_separate(spdk_bdev_desc_get_bdev(desc))) {
return -EINVAL;
}
if (!_bdev_io_check_md_buf(&iov, md_buf)) {
return -EINVAL;
}
return bdev_read_blocks_with_md(desc, ch, buf, md_buf, offset_blocks, num_blocks,
cb, cb_arg);
}
int
spdk_bdev_readv(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt,
uint64_t offset, uint64_t nbytes,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
uint64_t offset_blocks, num_blocks;
if (bdev_bytes_to_blocks(spdk_bdev_desc_get_bdev(desc), offset, &offset_blocks,
nbytes, &num_blocks) != 0) {
return -EINVAL;
}
return spdk_bdev_readv_blocks(desc, ch, iov, iovcnt, offset_blocks, num_blocks, cb, cb_arg);
}
static int
bdev_readv_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt, void *md_buf, uint64_t offset_blocks,
uint64_t num_blocks, spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
return -EINVAL;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_READ;
bdev_io->u.bdev.iovs = iov;
bdev_io->u.bdev.iovcnt = iovcnt;
bdev_io->u.bdev.md_buf = md_buf;
bdev_io->u.bdev.num_blocks = num_blocks;
bdev_io->u.bdev.offset_blocks = offset_blocks;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
bdev_io_submit(bdev_io);
return 0;
}
int spdk_bdev_readv_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt,
uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
return bdev_readv_blocks_with_md(desc, ch, iov, iovcnt, NULL, offset_blocks,
num_blocks, cb, cb_arg);
}
int
spdk_bdev_readv_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt, void *md_buf,
uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
if (!spdk_bdev_is_md_separate(spdk_bdev_desc_get_bdev(desc))) {
return -EINVAL;
}
if (!_bdev_io_check_md_buf(iov, md_buf)) {
return -EINVAL;
}
return bdev_readv_blocks_with_md(desc, ch, iov, iovcnt, md_buf, offset_blocks,
num_blocks, cb, cb_arg);
}
static int
bdev_write_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
void *buf, void *md_buf, uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!desc->write) {
return -EBADF;
}
if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
return -EINVAL;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE;
bdev_io->u.bdev.iovs = &bdev_io->iov;
bdev_io->u.bdev.iovs[0].iov_base = buf;
bdev_io->u.bdev.iovs[0].iov_len = num_blocks * bdev->blocklen;
bdev_io->u.bdev.iovcnt = 1;
bdev_io->u.bdev.md_buf = md_buf;
bdev_io->u.bdev.num_blocks = num_blocks;
bdev_io->u.bdev.offset_blocks = offset_blocks;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
bdev_io_submit(bdev_io);
return 0;
}
int
spdk_bdev_write(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
void *buf, uint64_t offset, uint64_t nbytes,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
uint64_t offset_blocks, num_blocks;
if (bdev_bytes_to_blocks(spdk_bdev_desc_get_bdev(desc), offset, &offset_blocks,
nbytes, &num_blocks) != 0) {
return -EINVAL;
}
return spdk_bdev_write_blocks(desc, ch, buf, offset_blocks, num_blocks, cb, cb_arg);
}
int
spdk_bdev_write_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
void *buf, uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
return bdev_write_blocks_with_md(desc, ch, buf, NULL, offset_blocks, num_blocks,
cb, cb_arg);
}
int
spdk_bdev_write_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
void *buf, void *md_buf, uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct iovec iov = {
.iov_base = buf,
};
if (!spdk_bdev_is_md_separate(spdk_bdev_desc_get_bdev(desc))) {
return -EINVAL;
}
if (!_bdev_io_check_md_buf(&iov, md_buf)) {
return -EINVAL;
}
return bdev_write_blocks_with_md(desc, ch, buf, md_buf, offset_blocks, num_blocks,
cb, cb_arg);
}
static int
bdev_writev_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt, void *md_buf,
uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!desc->write) {
return -EBADF;
}
if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
return -EINVAL;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE;
bdev_io->u.bdev.iovs = iov;
bdev_io->u.bdev.iovcnt = iovcnt;
bdev_io->u.bdev.md_buf = md_buf;
bdev_io->u.bdev.num_blocks = num_blocks;
bdev_io->u.bdev.offset_blocks = offset_blocks;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
bdev_io_submit(bdev_io);
return 0;
}
int
spdk_bdev_writev(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt,
uint64_t offset, uint64_t len,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
uint64_t offset_blocks, num_blocks;
if (bdev_bytes_to_blocks(spdk_bdev_desc_get_bdev(desc), offset, &offset_blocks,
len, &num_blocks) != 0) {
return -EINVAL;
}
return spdk_bdev_writev_blocks(desc, ch, iov, iovcnt, offset_blocks, num_blocks, cb, cb_arg);
}
int
spdk_bdev_writev_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt,
uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
return bdev_writev_blocks_with_md(desc, ch, iov, iovcnt, NULL, offset_blocks,
num_blocks, cb, cb_arg);
}
int
spdk_bdev_writev_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt, void *md_buf,
uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
if (!spdk_bdev_is_md_separate(spdk_bdev_desc_get_bdev(desc))) {
return -EINVAL;
}
if (!_bdev_io_check_md_buf(iov, md_buf)) {
return -EINVAL;
}
return bdev_writev_blocks_with_md(desc, ch, iov, iovcnt, md_buf, offset_blocks,
num_blocks, cb, cb_arg);
}
static void
bdev_compare_do_read_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
{
struct spdk_bdev_io *parent_io = cb_arg;
uint8_t *read_buf = bdev_io->u.bdev.iovs[0].iov_base;
int i, rc = 0;
if (!success) {
parent_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
parent_io->internal.cb(parent_io, false, parent_io->internal.caller_ctx);
spdk_bdev_free_io(bdev_io);
return;
}
for (i = 0; i < parent_io->u.bdev.iovcnt; i++) {
rc = memcmp(read_buf,
parent_io->u.bdev.iovs[i].iov_base,
parent_io->u.bdev.iovs[i].iov_len);
if (rc) {
break;
}
read_buf += parent_io->u.bdev.iovs[i].iov_len;
}
spdk_bdev_free_io(bdev_io);
if (rc == 0) {
parent_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
parent_io->internal.cb(parent_io, true, parent_io->internal.caller_ctx);
} else {
parent_io->internal.status = SPDK_BDEV_IO_STATUS_MISCOMPARE;
parent_io->internal.cb(parent_io, false, parent_io->internal.caller_ctx);
}
}
static void
bdev_compare_do_read(void *_bdev_io)
{
struct spdk_bdev_io *bdev_io = _bdev_io;
int rc;
rc = spdk_bdev_read_blocks(bdev_io->internal.desc,
spdk_io_channel_from_ctx(bdev_io->internal.ch), NULL,
bdev_io->u.bdev.offset_blocks, bdev_io->u.bdev.num_blocks,
bdev_compare_do_read_done, bdev_io);
if (rc == -ENOMEM) {
bdev_queue_io_wait_with_cb(bdev_io, bdev_compare_do_read);
} else if (rc != 0) {
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
}
}
static int
bdev_comparev_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt, void *md_buf,
uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
return -EINVAL;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_COMPARE;
bdev_io->u.bdev.iovs = iov;
bdev_io->u.bdev.iovcnt = iovcnt;
bdev_io->u.bdev.md_buf = md_buf;
bdev_io->u.bdev.num_blocks = num_blocks;
bdev_io->u.bdev.offset_blocks = offset_blocks;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
if (bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_COMPARE)) {
bdev_io_submit(bdev_io);
return 0;
}
bdev_compare_do_read(bdev_io);
return 0;
}
int
spdk_bdev_comparev_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt,
uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
return bdev_comparev_blocks_with_md(desc, ch, iov, iovcnt, NULL, offset_blocks,
num_blocks, cb, cb_arg);
}
int
spdk_bdev_comparev_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt, void *md_buf,
uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
if (!spdk_bdev_is_md_separate(spdk_bdev_desc_get_bdev(desc))) {
return -EINVAL;
}
if (!_bdev_io_check_md_buf(iov, md_buf)) {
return -EINVAL;
}
return bdev_comparev_blocks_with_md(desc, ch, iov, iovcnt, md_buf, offset_blocks,
num_blocks, cb, cb_arg);
}
static int
bdev_compare_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
void *buf, void *md_buf, uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
return -EINVAL;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_COMPARE;
bdev_io->u.bdev.iovs = &bdev_io->iov;
bdev_io->u.bdev.iovs[0].iov_base = buf;
bdev_io->u.bdev.iovs[0].iov_len = num_blocks * bdev->blocklen;
bdev_io->u.bdev.iovcnt = 1;
bdev_io->u.bdev.md_buf = md_buf;
bdev_io->u.bdev.num_blocks = num_blocks;
bdev_io->u.bdev.offset_blocks = offset_blocks;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
if (bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_COMPARE)) {
bdev_io_submit(bdev_io);
return 0;
}
bdev_compare_do_read(bdev_io);
return 0;
}
int
spdk_bdev_compare_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
void *buf, uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
return bdev_compare_blocks_with_md(desc, ch, buf, NULL, offset_blocks, num_blocks,
cb, cb_arg);
}
int
spdk_bdev_compare_blocks_with_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
void *buf, void *md_buf, uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct iovec iov = {
.iov_base = buf,
};
if (!spdk_bdev_is_md_separate(spdk_bdev_desc_get_bdev(desc))) {
return -EINVAL;
}
if (!_bdev_io_check_md_buf(&iov, md_buf)) {
return -EINVAL;
}
return bdev_compare_blocks_with_md(desc, ch, buf, md_buf, offset_blocks, num_blocks,
cb, cb_arg);
}
static void
bdev_comparev_and_writev_blocks_unlocked(void *ctx, int unlock_status)
{
struct spdk_bdev_io *bdev_io = ctx;
if (unlock_status) {
SPDK_ERRLOG("LBA range unlock failed\n");
}
bdev_io->internal.cb(bdev_io, bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS ? true :
false, bdev_io->internal.caller_ctx);
}
static void
bdev_comparev_and_writev_blocks_unlock(struct spdk_bdev_io *bdev_io, int status)
{
bdev_io->internal.status = status;
bdev_unlock_lba_range(bdev_io->internal.desc, spdk_io_channel_from_ctx(bdev_io->internal.ch),
bdev_io->u.bdev.offset_blocks, bdev_io->u.bdev.num_blocks,
bdev_comparev_and_writev_blocks_unlocked, bdev_io);
}
static void
bdev_compare_and_write_do_write_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
{
struct spdk_bdev_io *parent_io = cb_arg;
if (!success) {
SPDK_ERRLOG("Compare and write operation failed\n");
}
spdk_bdev_free_io(bdev_io);
bdev_comparev_and_writev_blocks_unlock(parent_io,
success ? SPDK_BDEV_IO_STATUS_SUCCESS : SPDK_BDEV_IO_STATUS_FAILED);
}
static void
bdev_compare_and_write_do_write(void *_bdev_io)
{
struct spdk_bdev_io *bdev_io = _bdev_io;
int rc;
rc = spdk_bdev_writev_blocks(bdev_io->internal.desc,
spdk_io_channel_from_ctx(bdev_io->internal.ch),
bdev_io->u.bdev.fused_iovs, bdev_io->u.bdev.fused_iovcnt,
bdev_io->u.bdev.offset_blocks, bdev_io->u.bdev.num_blocks,
bdev_compare_and_write_do_write_done, bdev_io);
if (rc == -ENOMEM) {
bdev_queue_io_wait_with_cb(bdev_io, bdev_compare_and_write_do_write);
} else if (rc != 0) {
bdev_comparev_and_writev_blocks_unlock(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
}
}
static void
bdev_compare_and_write_do_compare_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
{
struct spdk_bdev_io *parent_io = cb_arg;
spdk_bdev_free_io(bdev_io);
if (!success) {
bdev_comparev_and_writev_blocks_unlock(parent_io, SPDK_BDEV_IO_STATUS_MISCOMPARE);
return;
}
bdev_compare_and_write_do_write(parent_io);
}
static void
bdev_compare_and_write_do_compare(void *_bdev_io)
{
struct spdk_bdev_io *bdev_io = _bdev_io;
int rc;
rc = spdk_bdev_comparev_blocks(bdev_io->internal.desc,
spdk_io_channel_from_ctx(bdev_io->internal.ch), bdev_io->u.bdev.iovs,
bdev_io->u.bdev.iovcnt, bdev_io->u.bdev.offset_blocks, bdev_io->u.bdev.num_blocks,
bdev_compare_and_write_do_compare_done, bdev_io);
if (rc == -ENOMEM) {
bdev_queue_io_wait_with_cb(bdev_io, bdev_compare_and_write_do_compare);
} else if (rc != 0) {
bdev_comparev_and_writev_blocks_unlock(bdev_io, SPDK_BDEV_IO_STATUS_FIRST_FUSED_FAILED);
}
}
static void
bdev_comparev_and_writev_blocks_locked(void *ctx, int status)
{
struct spdk_bdev_io *bdev_io = ctx;
if (status) {
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FIRST_FUSED_FAILED;
bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
return;
}
bdev_compare_and_write_do_compare(bdev_io);
}
int
spdk_bdev_comparev_and_writev_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *compare_iov, int compare_iovcnt,
struct iovec *write_iov, int write_iovcnt,
uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!desc->write) {
return -EBADF;
}
if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
return -EINVAL;
}
if (num_blocks > bdev->acwu) {
return -EINVAL;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE;
bdev_io->u.bdev.iovs = compare_iov;
bdev_io->u.bdev.iovcnt = compare_iovcnt;
bdev_io->u.bdev.fused_iovs = write_iov;
bdev_io->u.bdev.fused_iovcnt = write_iovcnt;
bdev_io->u.bdev.md_buf = NULL;
bdev_io->u.bdev.num_blocks = num_blocks;
bdev_io->u.bdev.offset_blocks = offset_blocks;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
if (bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_COMPARE_AND_WRITE)) {
bdev_io_submit(bdev_io);
return 0;
}
return bdev_lock_lba_range(desc, ch, offset_blocks, num_blocks,
bdev_comparev_and_writev_blocks_locked, bdev_io);
}
int
spdk_bdev_zcopy_start(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
struct iovec *iov, int iovcnt,
uint64_t offset_blocks, uint64_t num_blocks,
bool populate,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!desc->write) {
return -EBADF;
}
if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
return -EINVAL;
}
if (!spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_ZCOPY)) {
return -ENOTSUP;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_ZCOPY;
bdev_io->u.bdev.num_blocks = num_blocks;
bdev_io->u.bdev.offset_blocks = offset_blocks;
bdev_io->u.bdev.iovs = iov;
bdev_io->u.bdev.iovcnt = iovcnt;
bdev_io->u.bdev.md_buf = NULL;
bdev_io->u.bdev.zcopy.populate = populate ? 1 : 0;
bdev_io->u.bdev.zcopy.commit = 0;
bdev_io->u.bdev.zcopy.start = 1;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
bdev_io_submit(bdev_io);
return 0;
}
int
spdk_bdev_zcopy_end(struct spdk_bdev_io *bdev_io, bool commit,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
if (bdev_io->type != SPDK_BDEV_IO_TYPE_ZCOPY) {
return -EINVAL;
}
bdev_io->u.bdev.zcopy.commit = commit ? 1 : 0;
bdev_io->u.bdev.zcopy.start = 0;
bdev_io->internal.caller_ctx = cb_arg;
bdev_io->internal.cb = cb;
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_PENDING;
bdev_io_submit(bdev_io);
return 0;
}
int
spdk_bdev_write_zeroes(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
uint64_t offset, uint64_t len,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
uint64_t offset_blocks, num_blocks;
if (bdev_bytes_to_blocks(spdk_bdev_desc_get_bdev(desc), offset, &offset_blocks,
len, &num_blocks) != 0) {
return -EINVAL;
}
return spdk_bdev_write_zeroes_blocks(desc, ch, offset_blocks, num_blocks, cb, cb_arg);
}
int
spdk_bdev_write_zeroes_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!desc->write) {
return -EBADF;
}
if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
return -EINVAL;
}
if (!bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_WRITE_ZEROES) &&
!bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_WRITE)) {
return -ENOTSUP;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->type = SPDK_BDEV_IO_TYPE_WRITE_ZEROES;
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->u.bdev.offset_blocks = offset_blocks;
bdev_io->u.bdev.num_blocks = num_blocks;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
if (bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_WRITE_ZEROES)) {
bdev_io_submit(bdev_io);
return 0;
}
assert(bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_WRITE));
assert(_bdev_get_block_size_with_md(bdev) <= ZERO_BUFFER_SIZE);
bdev_io->u.bdev.split_remaining_num_blocks = num_blocks;
bdev_io->u.bdev.split_current_offset_blocks = offset_blocks;
bdev_write_zero_buffer_next(bdev_io);
return 0;
}
int
spdk_bdev_unmap(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
uint64_t offset, uint64_t nbytes,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
uint64_t offset_blocks, num_blocks;
if (bdev_bytes_to_blocks(spdk_bdev_desc_get_bdev(desc), offset, &offset_blocks,
nbytes, &num_blocks) != 0) {
return -EINVAL;
}
return spdk_bdev_unmap_blocks(desc, ch, offset_blocks, num_blocks, cb, cb_arg);
}
int
spdk_bdev_unmap_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!desc->write) {
return -EBADF;
}
if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
return -EINVAL;
}
if (num_blocks == 0) {
SPDK_ERRLOG("Can't unmap 0 bytes\n");
return -EINVAL;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_UNMAP;
bdev_io->u.bdev.iovs = &bdev_io->iov;
bdev_io->u.bdev.iovs[0].iov_base = NULL;
bdev_io->u.bdev.iovs[0].iov_len = 0;
bdev_io->u.bdev.iovcnt = 1;
bdev_io->u.bdev.offset_blocks = offset_blocks;
bdev_io->u.bdev.num_blocks = num_blocks;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
bdev_io_submit(bdev_io);
return 0;
}
int
spdk_bdev_flush(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
uint64_t offset, uint64_t length,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
uint64_t offset_blocks, num_blocks;
if (bdev_bytes_to_blocks(spdk_bdev_desc_get_bdev(desc), offset, &offset_blocks,
length, &num_blocks) != 0) {
return -EINVAL;
}
return spdk_bdev_flush_blocks(desc, ch, offset_blocks, num_blocks, cb, cb_arg);
}
int
spdk_bdev_flush_blocks(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
uint64_t offset_blocks, uint64_t num_blocks,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!desc->write) {
return -EBADF;
}
if (!bdev_io_valid_blocks(bdev, offset_blocks, num_blocks)) {
return -EINVAL;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_FLUSH;
bdev_io->u.bdev.iovs = NULL;
bdev_io->u.bdev.iovcnt = 0;
bdev_io->u.bdev.offset_blocks = offset_blocks;
bdev_io->u.bdev.num_blocks = num_blocks;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
bdev_io_submit(bdev_io);
return 0;
}
static void
bdev_reset_dev(struct spdk_io_channel_iter *i, int status)
{
struct spdk_bdev_channel *ch = spdk_io_channel_iter_get_ctx(i);
struct spdk_bdev_io *bdev_io;
bdev_io = TAILQ_FIRST(&ch->queued_resets);
TAILQ_REMOVE(&ch->queued_resets, bdev_io, internal.link);
bdev_io_submit_reset(bdev_io);
}
static void
bdev_reset_freeze_channel(struct spdk_io_channel_iter *i)
{
struct spdk_io_channel *ch;
struct spdk_bdev_channel *channel;
struct spdk_bdev_mgmt_channel *mgmt_channel;
struct spdk_bdev_shared_resource *shared_resource;
bdev_io_tailq_t tmp_queued;
TAILQ_INIT(&tmp_queued);
ch = spdk_io_channel_iter_get_channel(i);
channel = spdk_io_channel_get_ctx(ch);
shared_resource = channel->shared_resource;
mgmt_channel = shared_resource->mgmt_ch;
channel->flags |= BDEV_CH_RESET_IN_PROGRESS;
if ((channel->flags & BDEV_CH_QOS_ENABLED) != 0) {
/* The QoS object is always valid and readable while
* the channel flag is set, so the lock here should not
* be necessary. We're not in the fast path though, so
* just take it anyway. */
pthread_mutex_lock(&channel->bdev->internal.mutex);
if (channel->bdev->internal.qos->ch == channel) {
TAILQ_SWAP(&channel->bdev->internal.qos->queued, &tmp_queued, spdk_bdev_io, internal.link);
}
pthread_mutex_unlock(&channel->bdev->internal.mutex);
}
bdev_abort_all_queued_io(&shared_resource->nomem_io, channel);
bdev_abort_all_buf_io(&mgmt_channel->need_buf_small, channel);
bdev_abort_all_buf_io(&mgmt_channel->need_buf_large, channel);
bdev_abort_all_queued_io(&tmp_queued, channel);
spdk_for_each_channel_continue(i, 0);
}
static void
bdev_start_reset(void *ctx)
{
struct spdk_bdev_channel *ch = ctx;
spdk_for_each_channel(__bdev_to_io_dev(ch->bdev), bdev_reset_freeze_channel,
ch, bdev_reset_dev);
}
static void
bdev_channel_start_reset(struct spdk_bdev_channel *ch)
{
struct spdk_bdev *bdev = ch->bdev;
assert(!TAILQ_EMPTY(&ch->queued_resets));
pthread_mutex_lock(&bdev->internal.mutex);
if (bdev->internal.reset_in_progress == NULL) {
bdev->internal.reset_in_progress = TAILQ_FIRST(&ch->queued_resets);
/*
* Take a channel reference for the target bdev for the life of this
* reset. This guards against the channel getting destroyed while
* spdk_for_each_channel() calls related to this reset IO are in
* progress. We will release the reference when this reset is
* completed.
*/
bdev->internal.reset_in_progress->u.reset.ch_ref = spdk_get_io_channel(__bdev_to_io_dev(bdev));
bdev_start_reset(ch);
}
pthread_mutex_unlock(&bdev->internal.mutex);
}
int
spdk_bdev_reset(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->internal.submit_tsc = spdk_get_ticks();
bdev_io->type = SPDK_BDEV_IO_TYPE_RESET;
bdev_io->u.reset.ch_ref = NULL;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
pthread_mutex_lock(&bdev->internal.mutex);
TAILQ_INSERT_TAIL(&channel->queued_resets, bdev_io, internal.link);
pthread_mutex_unlock(&bdev->internal.mutex);
TAILQ_INSERT_TAIL(&bdev_io->internal.ch->io_submitted, bdev_io,
internal.ch_link);
bdev_channel_start_reset(channel);
return 0;
}
void
spdk_bdev_get_io_stat(struct spdk_bdev *bdev, struct spdk_io_channel *ch,
struct spdk_bdev_io_stat *stat)
{
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
*stat = channel->stat;
}
static void
bdev_get_device_stat_done(struct spdk_io_channel_iter *i, int status)
{
void *io_device = spdk_io_channel_iter_get_io_device(i);
struct spdk_bdev_iostat_ctx *bdev_iostat_ctx = spdk_io_channel_iter_get_ctx(i);
bdev_iostat_ctx->cb(__bdev_from_io_dev(io_device), bdev_iostat_ctx->stat,
bdev_iostat_ctx->cb_arg, 0);
free(bdev_iostat_ctx);
}
static void
bdev_get_each_channel_stat(struct spdk_io_channel_iter *i)
{
struct spdk_bdev_iostat_ctx *bdev_iostat_ctx = spdk_io_channel_iter_get_ctx(i);
struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
bdev_io_stat_add(bdev_iostat_ctx->stat, &channel->stat);
spdk_for_each_channel_continue(i, 0);
}
void
spdk_bdev_get_device_stat(struct spdk_bdev *bdev, struct spdk_bdev_io_stat *stat,
spdk_bdev_get_device_stat_cb cb, void *cb_arg)
{
struct spdk_bdev_iostat_ctx *bdev_iostat_ctx;
assert(bdev != NULL);
assert(stat != NULL);
assert(cb != NULL);
bdev_iostat_ctx = calloc(1, sizeof(struct spdk_bdev_iostat_ctx));
if (bdev_iostat_ctx == NULL) {
SPDK_ERRLOG("Unable to allocate memory for spdk_bdev_iostat_ctx\n");
cb(bdev, stat, cb_arg, -ENOMEM);
return;
}
bdev_iostat_ctx->stat = stat;
bdev_iostat_ctx->cb = cb;
bdev_iostat_ctx->cb_arg = cb_arg;
/* Start with the statistics from previously deleted channels. */
pthread_mutex_lock(&bdev->internal.mutex);
bdev_io_stat_add(bdev_iostat_ctx->stat, &bdev->internal.stat);
pthread_mutex_unlock(&bdev->internal.mutex);
/* Then iterate and add the statistics from each existing channel. */
spdk_for_each_channel(__bdev_to_io_dev(bdev),
bdev_get_each_channel_stat,
bdev_iostat_ctx,
bdev_get_device_stat_done);
}
int
spdk_bdev_nvme_admin_passthru(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
const struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!desc->write) {
return -EBADF;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_NVME_ADMIN;
bdev_io->u.nvme_passthru.cmd = *cmd;
bdev_io->u.nvme_passthru.buf = buf;
bdev_io->u.nvme_passthru.nbytes = nbytes;
bdev_io->u.nvme_passthru.md_buf = NULL;
bdev_io->u.nvme_passthru.md_len = 0;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
bdev_io_submit(bdev_io);
return 0;
}
int
spdk_bdev_nvme_io_passthru(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
const struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!desc->write) {
/*
* Do not try to parse the NVMe command - we could maybe use bits in the opcode
* to easily determine if the command is a read or write, but for now just
* do not allow io_passthru with a read-only descriptor.
*/
return -EBADF;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_NVME_IO;
bdev_io->u.nvme_passthru.cmd = *cmd;
bdev_io->u.nvme_passthru.buf = buf;
bdev_io->u.nvme_passthru.nbytes = nbytes;
bdev_io->u.nvme_passthru.md_buf = NULL;
bdev_io->u.nvme_passthru.md_len = 0;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
bdev_io_submit(bdev_io);
return 0;
}
int
spdk_bdev_nvme_io_passthru_md(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
const struct spdk_nvme_cmd *cmd, void *buf, size_t nbytes, void *md_buf, size_t md_len,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
if (!desc->write) {
/*
* Do not try to parse the NVMe command - we could maybe use bits in the opcode
* to easily determine if the command is a read or write, but for now just
* do not allow io_passthru with a read-only descriptor.
*/
return -EBADF;
}
bdev_io = bdev_channel_get_io(channel);
if (!bdev_io) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_NVME_IO_MD;
bdev_io->u.nvme_passthru.cmd = *cmd;
bdev_io->u.nvme_passthru.buf = buf;
bdev_io->u.nvme_passthru.nbytes = nbytes;
bdev_io->u.nvme_passthru.md_buf = md_buf;
bdev_io->u.nvme_passthru.md_len = md_len;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
bdev_io_submit(bdev_io);
return 0;
}
static void bdev_abort_retry(void *ctx);
static void bdev_abort(struct spdk_bdev_io *parent_io);
static void
bdev_abort_io_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
{
struct spdk_bdev_channel *channel = bdev_io->internal.ch;
struct spdk_bdev_io *parent_io = cb_arg;
struct spdk_bdev_io *bio_to_abort, *tmp_io;
bio_to_abort = bdev_io->u.abort.bio_to_abort;
spdk_bdev_free_io(bdev_io);
if (!success) {
/* Check if the target I/O completed in the meantime. */
TAILQ_FOREACH(tmp_io, &channel->io_submitted, internal.ch_link) {
if (tmp_io == bio_to_abort) {
break;
}
}
/* If the target I/O still exists, set the parent to failed. */
if (tmp_io != NULL) {
parent_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
}
}
parent_io->u.bdev.split_outstanding--;
if (parent_io->u.bdev.split_outstanding == 0) {
if (parent_io->internal.status == SPDK_BDEV_IO_STATUS_NOMEM) {
bdev_abort_retry(parent_io);
} else {
bdev_io_complete(parent_io);
}
}
}
static int
bdev_abort_io(struct spdk_bdev_desc *desc, struct spdk_bdev_channel *channel,
struct spdk_bdev_io *bio_to_abort,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_io *bdev_io;
if (bio_to_abort->type == SPDK_BDEV_IO_TYPE_ABORT ||
bio_to_abort->type == SPDK_BDEV_IO_TYPE_RESET) {
/* TODO: Abort reset or abort request. */
return -ENOTSUP;
}
bdev_io = bdev_channel_get_io(channel);
if (bdev_io == NULL) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->type = SPDK_BDEV_IO_TYPE_ABORT;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
if (bdev->split_on_optimal_io_boundary && bdev_io_should_split(bio_to_abort)) {
bdev_io->u.bdev.abort.bio_cb_arg = bio_to_abort;
/* Parent abort request is not submitted directly, but to manage its
* execution add it to the submitted list here.
*/
bdev_io->internal.submit_tsc = spdk_get_ticks();
TAILQ_INSERT_TAIL(&channel->io_submitted, bdev_io, internal.ch_link);
bdev_abort(bdev_io);
return 0;
}
bdev_io->u.abort.bio_to_abort = bio_to_abort;
/* Submit the abort request to the underlying bdev module. */
bdev_io_submit(bdev_io);
return 0;
}
static uint32_t
_bdev_abort(struct spdk_bdev_io *parent_io)
{
struct spdk_bdev_desc *desc = parent_io->internal.desc;
struct spdk_bdev_channel *channel = parent_io->internal.ch;
void *bio_cb_arg;
struct spdk_bdev_io *bio_to_abort;
uint32_t matched_ios;
int rc;
bio_cb_arg = parent_io->u.bdev.abort.bio_cb_arg;
/* matched_ios is returned and will be kept by the caller.
*
* This funcion will be used for two cases, 1) the same cb_arg is used for
* multiple I/Os, 2) a single large I/O is split into smaller ones.
* Incrementing split_outstanding directly here may confuse readers especially
* for the 1st case.
*
* Completion of I/O abort is processed after stack unwinding. Hence this trick
* works as expected.
*/
matched_ios = 0;
parent_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
TAILQ_FOREACH(bio_to_abort, &channel->io_submitted, internal.ch_link) {
if (bio_to_abort->internal.caller_ctx != bio_cb_arg) {
continue;
}
if (bio_to_abort->internal.submit_tsc > parent_io->internal.submit_tsc) {
/* Any I/O which was submitted after this abort command should be excluded. */
continue;
}
rc = bdev_abort_io(desc, channel, bio_to_abort, bdev_abort_io_done, parent_io);
if (rc != 0) {
if (rc == -ENOMEM) {
parent_io->internal.status = SPDK_BDEV_IO_STATUS_NOMEM;
} else {
parent_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
}
break;
}
matched_ios++;
}
return matched_ios;
}
static void
bdev_abort_retry(void *ctx)
{
struct spdk_bdev_io *parent_io = ctx;
uint32_t matched_ios;
matched_ios = _bdev_abort(parent_io);
if (matched_ios == 0) {
if (parent_io->internal.status == SPDK_BDEV_IO_STATUS_NOMEM) {
bdev_queue_io_wait_with_cb(parent_io, bdev_abort_retry);
} else {
/* For retry, the case that no target I/O was found is success
* because it means target I/Os completed in the meantime.
*/
bdev_io_complete(parent_io);
}
return;
}
/* Use split_outstanding to manage the progress of aborting I/Os. */
parent_io->u.bdev.split_outstanding = matched_ios;
}
static void
bdev_abort(struct spdk_bdev_io *parent_io)
{
uint32_t matched_ios;
matched_ios = _bdev_abort(parent_io);
if (matched_ios == 0) {
if (parent_io->internal.status == SPDK_BDEV_IO_STATUS_NOMEM) {
bdev_queue_io_wait_with_cb(parent_io, bdev_abort_retry);
} else {
/* The case the no target I/O was found is failure. */
parent_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
bdev_io_complete(parent_io);
}
return;
}
/* Use split_outstanding to manage the progress of aborting I/Os. */
parent_io->u.bdev.split_outstanding = matched_ios;
}
int
spdk_bdev_abort(struct spdk_bdev_desc *desc, struct spdk_io_channel *ch,
void *bio_cb_arg,
spdk_bdev_io_completion_cb cb, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
struct spdk_bdev_io *bdev_io;
if (bio_cb_arg == NULL) {
return -EINVAL;
}
if (!spdk_bdev_io_type_supported(bdev, SPDK_BDEV_IO_TYPE_ABORT)) {
return -ENOTSUP;
}
bdev_io = bdev_channel_get_io(channel);
if (bdev_io == NULL) {
return -ENOMEM;
}
bdev_io->internal.ch = channel;
bdev_io->internal.desc = desc;
bdev_io->internal.submit_tsc = spdk_get_ticks();
bdev_io->type = SPDK_BDEV_IO_TYPE_ABORT;
bdev_io_init(bdev_io, bdev, cb_arg, cb);
bdev_io->u.bdev.abort.bio_cb_arg = bio_cb_arg;
/* Parent abort request is not submitted directly, but to manage its execution,
* add it to the submitted list here.
*/
TAILQ_INSERT_TAIL(&channel->io_submitted, bdev_io, internal.ch_link);
bdev_abort(bdev_io);
return 0;
}
int
spdk_bdev_queue_io_wait(struct spdk_bdev *bdev, struct spdk_io_channel *ch,
struct spdk_bdev_io_wait_entry *entry)
{
struct spdk_bdev_channel *channel = spdk_io_channel_get_ctx(ch);
struct spdk_bdev_mgmt_channel *mgmt_ch = channel->shared_resource->mgmt_ch;
if (bdev != entry->bdev) {
SPDK_ERRLOG("bdevs do not match\n");
return -EINVAL;
}
if (mgmt_ch->per_thread_cache_count > 0) {
SPDK_ERRLOG("Cannot queue io_wait if spdk_bdev_io available in per-thread cache\n");
return -EINVAL;
}
TAILQ_INSERT_TAIL(&mgmt_ch->io_wait_queue, entry, link);
return 0;
}
static void
bdev_ch_retry_io(struct spdk_bdev_channel *bdev_ch)
{
struct spdk_bdev *bdev = bdev_ch->bdev;
struct spdk_bdev_shared_resource *shared_resource = bdev_ch->shared_resource;
struct spdk_bdev_io *bdev_io;
if (shared_resource->io_outstanding > shared_resource->nomem_threshold) {
/*
* Allow some more I/O to complete before retrying the nomem_io queue.
* Some drivers (such as nvme) cannot immediately take a new I/O in
* the context of a completion, because the resources for the I/O are
* not released until control returns to the bdev poller. Also, we
* may require several small I/O to complete before a larger I/O
* (that requires splitting) can be submitted.
*/
return;
}
while (!TAILQ_EMPTY(&shared_resource->nomem_io)) {
bdev_io = TAILQ_FIRST(&shared_resource->nomem_io);
TAILQ_REMOVE(&shared_resource->nomem_io, bdev_io, internal.link);
bdev_io->internal.ch->io_outstanding++;
shared_resource->io_outstanding++;
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_PENDING;
bdev_io->internal.error.nvme.cdw0 = 0;
bdev_io->num_retries++;
bdev->fn_table->submit_request(spdk_bdev_io_get_io_channel(bdev_io), bdev_io);
if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_NOMEM) {
break;
}
}
}
static inline void
bdev_io_complete(void *ctx)
{
struct spdk_bdev_io *bdev_io = ctx;
struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch;
uint64_t tsc, tsc_diff;
if (spdk_unlikely(bdev_io->internal.in_submit_request || bdev_io->internal.io_submit_ch)) {
/*
* Send the completion to the thread that originally submitted the I/O,
* which may not be the current thread in the case of QoS.
*/
if (bdev_io->internal.io_submit_ch) {
bdev_io->internal.ch = bdev_io->internal.io_submit_ch;
bdev_io->internal.io_submit_ch = NULL;
}
/*
* Defer completion to avoid potential infinite recursion if the
* user's completion callback issues a new I/O.
*/
spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io),
bdev_io_complete, bdev_io);
return;
}
tsc = spdk_get_ticks();
tsc_diff = tsc - bdev_io->internal.submit_tsc;
spdk_trace_record_tsc(tsc, TRACE_BDEV_IO_DONE, 0, 0, (uintptr_t)bdev_io);
TAILQ_REMOVE(&bdev_ch->io_submitted, bdev_io, internal.ch_link);
if (bdev_io->internal.ch->histogram) {
spdk_histogram_data_tally(bdev_io->internal.ch->histogram, tsc_diff);
}
if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS) {
switch (bdev_io->type) {
case SPDK_BDEV_IO_TYPE_READ:
bdev_io->internal.ch->stat.bytes_read += bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
bdev_io->internal.ch->stat.num_read_ops++;
bdev_io->internal.ch->stat.read_latency_ticks += tsc_diff;
break;
case SPDK_BDEV_IO_TYPE_WRITE:
bdev_io->internal.ch->stat.bytes_written += bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
bdev_io->internal.ch->stat.num_write_ops++;
bdev_io->internal.ch->stat.write_latency_ticks += tsc_diff;
break;
case SPDK_BDEV_IO_TYPE_UNMAP:
bdev_io->internal.ch->stat.bytes_unmapped += bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
bdev_io->internal.ch->stat.num_unmap_ops++;
bdev_io->internal.ch->stat.unmap_latency_ticks += tsc_diff;
break;
case SPDK_BDEV_IO_TYPE_ZCOPY:
/* Track the data in the start phase only */
if (bdev_io->u.bdev.zcopy.start) {
if (bdev_io->u.bdev.zcopy.populate) {
bdev_io->internal.ch->stat.bytes_read +=
bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
bdev_io->internal.ch->stat.num_read_ops++;
bdev_io->internal.ch->stat.read_latency_ticks += tsc_diff;
} else {
bdev_io->internal.ch->stat.bytes_written +=
bdev_io->u.bdev.num_blocks * bdev_io->bdev->blocklen;
bdev_io->internal.ch->stat.num_write_ops++;
bdev_io->internal.ch->stat.write_latency_ticks += tsc_diff;
}
}
break;
default:
break;
}
}
#ifdef SPDK_CONFIG_VTUNE
uint64_t now_tsc = spdk_get_ticks();
if (now_tsc > (bdev_io->internal.ch->start_tsc + bdev_io->internal.ch->interval_tsc)) {
uint64_t data[5];
data[0] = bdev_io->internal.ch->stat.num_read_ops - bdev_io->internal.ch->prev_stat.num_read_ops;
data[1] = bdev_io->internal.ch->stat.bytes_read - bdev_io->internal.ch->prev_stat.bytes_read;
data[2] = bdev_io->internal.ch->stat.num_write_ops - bdev_io->internal.ch->prev_stat.num_write_ops;
data[3] = bdev_io->internal.ch->stat.bytes_written - bdev_io->internal.ch->prev_stat.bytes_written;
data[4] = bdev_io->bdev->fn_table->get_spin_time ?
bdev_io->bdev->fn_table->get_spin_time(spdk_bdev_io_get_io_channel(bdev_io)) : 0;
__itt_metadata_add(g_bdev_mgr.domain, __itt_null, bdev_io->internal.ch->handle,
__itt_metadata_u64, 5, data);
bdev_io->internal.ch->prev_stat = bdev_io->internal.ch->stat;
bdev_io->internal.ch->start_tsc = now_tsc;
}
#endif
assert(bdev_io->internal.cb != NULL);
assert(spdk_get_thread() == spdk_bdev_io_get_thread(bdev_io));
bdev_io->internal.cb(bdev_io, bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS,
bdev_io->internal.caller_ctx);
}
static void
bdev_reset_complete(struct spdk_io_channel_iter *i, int status)
{
struct spdk_bdev_io *bdev_io = spdk_io_channel_iter_get_ctx(i);
if (bdev_io->u.reset.ch_ref != NULL) {
spdk_put_io_channel(bdev_io->u.reset.ch_ref);
bdev_io->u.reset.ch_ref = NULL;
}
bdev_io_complete(bdev_io);
}
static void
bdev_unfreeze_channel(struct spdk_io_channel_iter *i)
{
struct spdk_bdev_io *bdev_io = spdk_io_channel_iter_get_ctx(i);
struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(_ch);
struct spdk_bdev_io *queued_reset;
ch->flags &= ~BDEV_CH_RESET_IN_PROGRESS;
while (!TAILQ_EMPTY(&ch->queued_resets)) {
queued_reset = TAILQ_FIRST(&ch->queued_resets);
TAILQ_REMOVE(&ch->queued_resets, queued_reset, internal.link);
spdk_bdev_io_complete(queued_reset, bdev_io->internal.status);
}
spdk_for_each_channel_continue(i, 0);
}
void
spdk_bdev_io_complete(struct spdk_bdev_io *bdev_io, enum spdk_bdev_io_status status)
{
struct spdk_bdev *bdev = bdev_io->bdev;
struct spdk_bdev_channel *bdev_ch = bdev_io->internal.ch;
struct spdk_bdev_shared_resource *shared_resource = bdev_ch->shared_resource;
bdev_io->internal.status = status;
if (spdk_unlikely(bdev_io->type == SPDK_BDEV_IO_TYPE_RESET)) {
bool unlock_channels = false;
if (status == SPDK_BDEV_IO_STATUS_NOMEM) {
SPDK_ERRLOG("NOMEM returned for reset\n");
}
pthread_mutex_lock(&bdev->internal.mutex);
if (bdev_io == bdev->internal.reset_in_progress) {
bdev->internal.reset_in_progress = NULL;
unlock_channels = true;
}
pthread_mutex_unlock(&bdev->internal.mutex);
if (unlock_channels) {
spdk_for_each_channel(__bdev_to_io_dev(bdev), bdev_unfreeze_channel,
bdev_io, bdev_reset_complete);
return;
}
} else {
_bdev_io_unset_bounce_buf(bdev_io);
assert(bdev_ch->io_outstanding > 0);
assert(shared_resource->io_outstanding > 0);
bdev_ch->io_outstanding--;
shared_resource->io_outstanding--;
if (spdk_unlikely(status == SPDK_BDEV_IO_STATUS_NOMEM)) {
TAILQ_INSERT_HEAD(&shared_resource->nomem_io, bdev_io, internal.link);
/*
* Wait for some of the outstanding I/O to complete before we
* retry any of the nomem_io. Normally we will wait for
* NOMEM_THRESHOLD_COUNT I/O to complete but for low queue
* depth channels we will instead wait for half to complete.
*/
shared_resource->nomem_threshold = spdk_max((int64_t)shared_resource->io_outstanding / 2,
(int64_t)shared_resource->io_outstanding - NOMEM_THRESHOLD_COUNT);
return;
}
if (spdk_unlikely(!TAILQ_EMPTY(&shared_resource->nomem_io))) {
bdev_ch_retry_io(bdev_ch);
}
}
bdev_io_complete(bdev_io);
}
void
spdk_bdev_io_complete_scsi_status(struct spdk_bdev_io *bdev_io, enum spdk_scsi_status sc,
enum spdk_scsi_sense sk, uint8_t asc, uint8_t ascq)
{
if (sc == SPDK_SCSI_STATUS_GOOD) {
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
} else {
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SCSI_ERROR;
bdev_io->internal.error.scsi.sc = sc;
bdev_io->internal.error.scsi.sk = sk;
bdev_io->internal.error.scsi.asc = asc;
bdev_io->internal.error.scsi.ascq = ascq;
}
spdk_bdev_io_complete(bdev_io, bdev_io->internal.status);
}
void
spdk_bdev_io_get_scsi_status(const struct spdk_bdev_io *bdev_io,
int *sc, int *sk, int *asc, int *ascq)
{
assert(sc != NULL);
assert(sk != NULL);
assert(asc != NULL);
assert(ascq != NULL);
switch (bdev_io->internal.status) {
case SPDK_BDEV_IO_STATUS_SUCCESS:
*sc = SPDK_SCSI_STATUS_GOOD;
*sk = SPDK_SCSI_SENSE_NO_SENSE;
*asc = SPDK_SCSI_ASC_NO_ADDITIONAL_SENSE;
*ascq = SPDK_SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
break;
case SPDK_BDEV_IO_STATUS_NVME_ERROR:
spdk_scsi_nvme_translate(bdev_io, sc, sk, asc, ascq);
break;
case SPDK_BDEV_IO_STATUS_SCSI_ERROR:
*sc = bdev_io->internal.error.scsi.sc;
*sk = bdev_io->internal.error.scsi.sk;
*asc = bdev_io->internal.error.scsi.asc;
*ascq = bdev_io->internal.error.scsi.ascq;
break;
default:
*sc = SPDK_SCSI_STATUS_CHECK_CONDITION;
*sk = SPDK_SCSI_SENSE_ABORTED_COMMAND;
*asc = SPDK_SCSI_ASC_NO_ADDITIONAL_SENSE;
*ascq = SPDK_SCSI_ASCQ_CAUSE_NOT_REPORTABLE;
break;
}
}
void
spdk_bdev_io_complete_aio_status(struct spdk_bdev_io *bdev_io, int aio_result)
{
if (aio_result == 0) {
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
} else {
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_AIO_ERROR;
}
bdev_io->internal.error.aio_result = aio_result;
spdk_bdev_io_complete(bdev_io, bdev_io->internal.status);
}
void
spdk_bdev_io_get_aio_status(const struct spdk_bdev_io *bdev_io, int *aio_result)
{
assert(aio_result != NULL);
if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_AIO_ERROR) {
*aio_result = bdev_io->internal.error.aio_result;
} else if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS) {
*aio_result = 0;
} else {
*aio_result = -EIO;
}
}
void
spdk_bdev_io_complete_nvme_status(struct spdk_bdev_io *bdev_io, uint32_t cdw0, int sct, int sc)
{
if (sct == SPDK_NVME_SCT_GENERIC && sc == SPDK_NVME_SC_SUCCESS) {
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
} else if (sct == SPDK_NVME_SCT_GENERIC && sc == SPDK_NVME_SC_ABORTED_BY_REQUEST) {
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_ABORTED;
} else {
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_NVME_ERROR;
}
bdev_io->internal.error.nvme.cdw0 = cdw0;
bdev_io->internal.error.nvme.sct = sct;
bdev_io->internal.error.nvme.sc = sc;
spdk_bdev_io_complete(bdev_io, bdev_io->internal.status);
}
void
spdk_bdev_io_get_nvme_status(const struct spdk_bdev_io *bdev_io, uint32_t *cdw0, int *sct, int *sc)
{
assert(sct != NULL);
assert(sc != NULL);
assert(cdw0 != NULL);
if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_NVME_ERROR) {
*sct = bdev_io->internal.error.nvme.sct;
*sc = bdev_io->internal.error.nvme.sc;
} else if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS) {
*sct = SPDK_NVME_SCT_GENERIC;
*sc = SPDK_NVME_SC_SUCCESS;
} else if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_ABORTED) {
*sct = SPDK_NVME_SCT_GENERIC;
*sc = SPDK_NVME_SC_ABORTED_BY_REQUEST;
} else {
*sct = SPDK_NVME_SCT_GENERIC;
*sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
}
*cdw0 = bdev_io->internal.error.nvme.cdw0;
}
void
spdk_bdev_io_get_nvme_fused_status(const struct spdk_bdev_io *bdev_io, uint32_t *cdw0,
int *first_sct, int *first_sc, int *second_sct, int *second_sc)
{
assert(first_sct != NULL);
assert(first_sc != NULL);
assert(second_sct != NULL);
assert(second_sc != NULL);
assert(cdw0 != NULL);
if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_NVME_ERROR) {
if (bdev_io->internal.error.nvme.sct == SPDK_NVME_SCT_MEDIA_ERROR &&
bdev_io->internal.error.nvme.sc == SPDK_NVME_SC_COMPARE_FAILURE) {
*first_sct = bdev_io->internal.error.nvme.sct;
*first_sc = bdev_io->internal.error.nvme.sc;
*second_sct = SPDK_NVME_SCT_GENERIC;
*second_sc = SPDK_NVME_SC_ABORTED_FAILED_FUSED;
} else {
*first_sct = SPDK_NVME_SCT_GENERIC;
*first_sc = SPDK_NVME_SC_SUCCESS;
*second_sct = bdev_io->internal.error.nvme.sct;
*second_sc = bdev_io->internal.error.nvme.sc;
}
} else if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_SUCCESS) {
*first_sct = SPDK_NVME_SCT_GENERIC;
*first_sc = SPDK_NVME_SC_SUCCESS;
*second_sct = SPDK_NVME_SCT_GENERIC;
*second_sc = SPDK_NVME_SC_SUCCESS;
} else if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_FIRST_FUSED_FAILED) {
*first_sct = SPDK_NVME_SCT_GENERIC;
*first_sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
*second_sct = SPDK_NVME_SCT_GENERIC;
*second_sc = SPDK_NVME_SC_ABORTED_FAILED_FUSED;
} else if (bdev_io->internal.status == SPDK_BDEV_IO_STATUS_MISCOMPARE) {
*first_sct = SPDK_NVME_SCT_MEDIA_ERROR;
*first_sc = SPDK_NVME_SC_COMPARE_FAILURE;
*second_sct = SPDK_NVME_SCT_GENERIC;
*second_sc = SPDK_NVME_SC_ABORTED_FAILED_FUSED;
} else {
*first_sct = SPDK_NVME_SCT_GENERIC;
*first_sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
*second_sct = SPDK_NVME_SCT_GENERIC;
*second_sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
}
*cdw0 = bdev_io->internal.error.nvme.cdw0;
}
struct spdk_thread *
spdk_bdev_io_get_thread(struct spdk_bdev_io *bdev_io)
{
return spdk_io_channel_get_thread(bdev_io->internal.ch->channel);
}
struct spdk_io_channel *
spdk_bdev_io_get_io_channel(struct spdk_bdev_io *bdev_io)
{
return bdev_io->internal.ch->channel;
}
static int
bdev_register(struct spdk_bdev *bdev)
{
char *bdev_name;
int ret;
assert(bdev->module != NULL);
if (!bdev->name) {
SPDK_ERRLOG("Bdev name is NULL\n");
return -EINVAL;
}
if (!strlen(bdev->name)) {
SPDK_ERRLOG("Bdev name must not be an empty string\n");
return -EINVAL;
}
/* Users often register their own I/O devices using the bdev name. In
* order to avoid conflicts, prepend bdev_. */
bdev_name = spdk_sprintf_alloc("bdev_%s", bdev->name);
if (!bdev_name) {
SPDK_ERRLOG("Unable to allocate memory for internal bdev name.\n");
return -ENOMEM;
}
bdev->internal.status = SPDK_BDEV_STATUS_READY;
bdev->internal.measured_queue_depth = UINT64_MAX;
bdev->internal.claim_module = NULL;
bdev->internal.qd_poller = NULL;
bdev->internal.qos = NULL;
ret = bdev_name_add(&bdev->internal.bdev_name, bdev, bdev->name);
if (ret != 0) {
free(bdev_name);
return ret;
}
/* If the user didn't specify a uuid, generate one. */
if (spdk_mem_all_zero(&bdev->uuid, sizeof(bdev->uuid))) {
spdk_uuid_generate(&bdev->uuid);
}
if (spdk_bdev_get_buf_align(bdev) > 1) {
if (bdev->split_on_optimal_io_boundary) {
bdev->optimal_io_boundary = spdk_min(bdev->optimal_io_boundary,
SPDK_BDEV_LARGE_BUF_MAX_SIZE / bdev->blocklen);
} else {
bdev->split_on_optimal_io_boundary = true;
bdev->optimal_io_boundary = SPDK_BDEV_LARGE_BUF_MAX_SIZE / bdev->blocklen;
}
}
/* If the user didn't specify a write unit size, set it to one. */
if (bdev->write_unit_size == 0) {
bdev->write_unit_size = 1;
}
/* Set ACWU value to 1 if bdev module did not set it (does not support it natively) */
if (bdev->acwu == 0) {
bdev->acwu = 1;
}
if (bdev->phys_blocklen == 0) {
bdev->phys_blocklen = spdk_bdev_get_data_block_size(bdev);
}
TAILQ_INIT(&bdev->internal.open_descs);
TAILQ_INIT(&bdev->internal.locked_ranges);
TAILQ_INIT(&bdev->internal.pending_locked_ranges);
TAILQ_INIT(&bdev->aliases);
bdev->internal.reset_in_progress = NULL;
spdk_io_device_register(__bdev_to_io_dev(bdev),
bdev_channel_create, bdev_channel_destroy,
sizeof(struct spdk_bdev_channel),
bdev_name);
free(bdev_name);
pthread_mutex_init(&bdev->internal.mutex, NULL);
SPDK_DEBUGLOG(bdev, "Inserting bdev %s into list\n", bdev->name);
TAILQ_INSERT_TAIL(&g_bdev_mgr.bdevs, bdev, internal.link);
return 0;
}
static void
bdev_destroy_cb(void *io_device)
{
int rc;
struct spdk_bdev *bdev;
spdk_bdev_unregister_cb cb_fn;
void *cb_arg;
bdev = __bdev_from_io_dev(io_device);
cb_fn = bdev->internal.unregister_cb;
cb_arg = bdev->internal.unregister_ctx;
pthread_mutex_destroy(&bdev->internal.mutex);
free(bdev->internal.qos);
rc = bdev->fn_table->destruct(bdev->ctxt);
if (rc < 0) {
SPDK_ERRLOG("destruct failed\n");
}
if (rc <= 0 && cb_fn != NULL) {
cb_fn(cb_arg, rc);
}
}
static void
bdev_register_finished(void *arg)
{
struct spdk_bdev *bdev = arg;
spdk_notify_send("bdev_register", spdk_bdev_get_name(bdev));
}
int
spdk_bdev_register(struct spdk_bdev *bdev)
{
int rc = bdev_register(bdev);
if (rc == 0) {
/* Examine configuration before initializing I/O */
bdev_examine(bdev);
spdk_bdev_wait_for_examine(bdev_register_finished, bdev);
}
return rc;
}
void
spdk_bdev_destruct_done(struct spdk_bdev *bdev, int bdeverrno)
{
if (bdev->internal.unregister_cb != NULL) {
bdev->internal.unregister_cb(bdev->internal.unregister_ctx, bdeverrno);
}
}
static void
_remove_notify(void *arg)
{
struct spdk_bdev_desc *desc = arg;
pthread_mutex_lock(&desc->mutex);
desc->refs--;
if (!desc->closed) {
pthread_mutex_unlock(&desc->mutex);
desc->callback.event_fn(SPDK_BDEV_EVENT_REMOVE, desc->bdev, desc->callback.ctx);
return;
} else if (0 == desc->refs) {
/* This descriptor was closed after this remove_notify message was sent.
* spdk_bdev_close() could not free the descriptor since this message was
* in flight, so we free it now using bdev_desc_free().
*/
pthread_mutex_unlock(&desc->mutex);
bdev_desc_free(desc);
return;
}
pthread_mutex_unlock(&desc->mutex);
}
/* Must be called while holding bdev->internal.mutex.
* returns: 0 - bdev removed and ready to be destructed.
* -EBUSY - bdev can't be destructed yet. */
static int
bdev_unregister_unsafe(struct spdk_bdev *bdev)
{
struct spdk_bdev_desc *desc, *tmp;
int rc = 0;
/* Notify each descriptor about hotremoval */
TAILQ_FOREACH_SAFE(desc, &bdev->internal.open_descs, link, tmp) {
rc = -EBUSY;
pthread_mutex_lock(&desc->mutex);
/*
* Defer invocation of the event_cb to a separate message that will
* run later on its thread. This ensures this context unwinds and
* we don't recursively unregister this bdev again if the event_cb
* immediately closes its descriptor.
*/
desc->refs++;
spdk_thread_send_msg(desc->thread, _remove_notify, desc);
pthread_mutex_unlock(&desc->mutex);
}
/* If there are no descriptors, proceed removing the bdev */
if (rc == 0) {
TAILQ_REMOVE(&g_bdev_mgr.bdevs, bdev, internal.link);
SPDK_DEBUGLOG(bdev, "Removing bdev %s from list done\n", bdev->name);
bdev_name_del(&bdev->internal.bdev_name);
spdk_notify_send("bdev_unregister", spdk_bdev_get_name(bdev));
}
return rc;
}
void
spdk_bdev_unregister(struct spdk_bdev *bdev, spdk_bdev_unregister_cb cb_fn, void *cb_arg)
{
struct spdk_thread *thread;
int rc;
SPDK_DEBUGLOG(bdev, "Removing bdev %s from list\n", bdev->name);
thread = spdk_get_thread();
if (!thread) {
/* The user called this from a non-SPDK thread. */
if (cb_fn != NULL) {
cb_fn(cb_arg, -ENOTSUP);
}
return;
}
pthread_mutex_lock(&g_bdev_mgr.mutex);
if (bdev->internal.status == SPDK_BDEV_STATUS_REMOVING) {
pthread_mutex_unlock(&g_bdev_mgr.mutex);
if (cb_fn) {
cb_fn(cb_arg, -EBUSY);
}
return;
}
pthread_mutex_lock(&bdev->internal.mutex);
bdev->internal.status = SPDK_BDEV_STATUS_REMOVING;
bdev->internal.unregister_cb = cb_fn;
bdev->internal.unregister_ctx = cb_arg;
/* Call under lock. */
rc = bdev_unregister_unsafe(bdev);
pthread_mutex_unlock(&bdev->internal.mutex);
pthread_mutex_unlock(&g_bdev_mgr.mutex);
if (rc == 0) {
spdk_io_device_unregister(__bdev_to_io_dev(bdev), bdev_destroy_cb);
}
}
static int
bdev_start_qos(struct spdk_bdev *bdev)
{
struct set_qos_limit_ctx *ctx;
/* Enable QoS */
if (bdev->internal.qos && bdev->internal.qos->thread == NULL) {
ctx = calloc(1, sizeof(*ctx));
if (ctx == NULL) {
SPDK_ERRLOG("Failed to allocate memory for QoS context\n");
return -ENOMEM;
}
ctx->bdev = bdev;
spdk_for_each_channel(__bdev_to_io_dev(bdev),
bdev_enable_qos_msg, ctx,
bdev_enable_qos_done);
}
return 0;
}
static int
bdev_open(struct spdk_bdev *bdev, bool write, struct spdk_bdev_desc *desc)
{
struct spdk_thread *thread;
int rc = 0;
thread = spdk_get_thread();
if (!thread) {
SPDK_ERRLOG("Cannot open bdev from non-SPDK thread.\n");
return -ENOTSUP;
}
SPDK_DEBUGLOG(bdev, "Opening descriptor %p for bdev %s on thread %p\n", desc, bdev->name,
spdk_get_thread());
desc->bdev = bdev;
desc->thread = thread;
desc->write = write;
pthread_mutex_lock(&bdev->internal.mutex);
if (bdev->internal.status == SPDK_BDEV_STATUS_REMOVING) {
pthread_mutex_unlock(&bdev->internal.mutex);
return -ENODEV;
}
if (write && bdev->internal.claim_module) {
SPDK_ERRLOG("Could not open %s - %s module already claimed it\n",
bdev->name, bdev->internal.claim_module->name);
pthread_mutex_unlock(&bdev->internal.mutex);
return -EPERM;
}
rc = bdev_start_qos(bdev);
if (rc != 0) {
SPDK_ERRLOG("Failed to start QoS on bdev %s\n", bdev->name);
pthread_mutex_unlock(&bdev->internal.mutex);
return rc;
}
TAILQ_INSERT_TAIL(&bdev->internal.open_descs, desc, link);
pthread_mutex_unlock(&bdev->internal.mutex);
return 0;
}
int
spdk_bdev_open_ext(const char *bdev_name, bool write, spdk_bdev_event_cb_t event_cb,
void *event_ctx, struct spdk_bdev_desc **_desc)
{
struct spdk_bdev_desc *desc;
struct spdk_bdev *bdev;
unsigned int event_id;
int rc;
if (event_cb == NULL) {
SPDK_ERRLOG("Missing event callback function\n");
return -EINVAL;
}
pthread_mutex_lock(&g_bdev_mgr.mutex);
bdev = bdev_get_by_name(bdev_name);
if (bdev == NULL) {
SPDK_NOTICELOG("Currently unable to find bdev with name: %s\n", bdev_name);
pthread_mutex_unlock(&g_bdev_mgr.mutex);
return -ENODEV;
}
desc = calloc(1, sizeof(*desc));
if (desc == NULL) {
SPDK_ERRLOG("Failed to allocate memory for bdev descriptor\n");
pthread_mutex_unlock(&g_bdev_mgr.mutex);
return -ENOMEM;
}
TAILQ_INIT(&desc->pending_media_events);
TAILQ_INIT(&desc->free_media_events);
desc->callback.event_fn = event_cb;
desc->callback.ctx = event_ctx;
pthread_mutex_init(&desc->mutex, NULL);
if (bdev->media_events) {
desc->media_events_buffer = calloc(MEDIA_EVENT_POOL_SIZE,
sizeof(*desc->media_events_buffer));
if (desc->media_events_buffer == NULL) {
SPDK_ERRLOG("Failed to initialize media event pool\n");
bdev_desc_free(desc);
pthread_mutex_unlock(&g_bdev_mgr.mutex);
return -ENOMEM;
}
for (event_id = 0; event_id < MEDIA_EVENT_POOL_SIZE; ++event_id) {
TAILQ_INSERT_TAIL(&desc->free_media_events,
&desc->media_events_buffer[event_id], tailq);
}
}
rc = bdev_open(bdev, write, desc);
if (rc != 0) {
bdev_desc_free(desc);
desc = NULL;
}
*_desc = desc;
pthread_mutex_unlock(&g_bdev_mgr.mutex);
return rc;
}
void
spdk_bdev_close(struct spdk_bdev_desc *desc)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
int rc;
SPDK_DEBUGLOG(bdev, "Closing descriptor %p for bdev %s on thread %p\n", desc, bdev->name,
spdk_get_thread());
assert(desc->thread == spdk_get_thread());
spdk_poller_unregister(&desc->io_timeout_poller);
pthread_mutex_lock(&g_bdev_mgr.mutex);
pthread_mutex_lock(&bdev->internal.mutex);
pthread_mutex_lock(&desc->mutex);
TAILQ_REMOVE(&bdev->internal.open_descs, desc, link);
desc->closed = true;
if (0 == desc->refs) {
pthread_mutex_unlock(&desc->mutex);
bdev_desc_free(desc);
} else {
pthread_mutex_unlock(&desc->mutex);
}
/* If no more descriptors, kill QoS channel */
if (bdev->internal.qos && TAILQ_EMPTY(&bdev->internal.open_descs)) {
SPDK_DEBUGLOG(bdev, "Closed last descriptor for bdev %s on thread %p. Stopping QoS.\n",
bdev->name, spdk_get_thread());
if (bdev_qos_destroy(bdev)) {
/* There isn't anything we can do to recover here. Just let the
* old QoS poller keep running. The QoS handling won't change
* cores when the user allocates a new channel, but it won't break. */
SPDK_ERRLOG("Unable to shut down QoS poller. It will continue running on the current thread.\n");
}
}
spdk_bdev_set_qd_sampling_period(bdev, 0);
if (bdev->internal.status == SPDK_BDEV_STATUS_REMOVING && TAILQ_EMPTY(&bdev->internal.open_descs)) {
rc = bdev_unregister_unsafe(bdev);
pthread_mutex_unlock(&bdev->internal.mutex);
pthread_mutex_unlock(&g_bdev_mgr.mutex);
if (rc == 0) {
spdk_io_device_unregister(__bdev_to_io_dev(bdev), bdev_destroy_cb);
}
} else {
pthread_mutex_unlock(&bdev->internal.mutex);
pthread_mutex_unlock(&g_bdev_mgr.mutex);
}
}
int
spdk_bdev_module_claim_bdev(struct spdk_bdev *bdev, struct spdk_bdev_desc *desc,
struct spdk_bdev_module *module)
{
if (bdev->internal.claim_module != NULL) {
SPDK_ERRLOG("bdev %s already claimed by module %s\n", bdev->name,
bdev->internal.claim_module->name);
return -EPERM;
}
if (desc && !desc->write) {
desc->write = true;
}
bdev->internal.claim_module = module;
return 0;
}
void
spdk_bdev_module_release_bdev(struct spdk_bdev *bdev)
{
assert(bdev->internal.claim_module != NULL);
bdev->internal.claim_module = NULL;
}
struct spdk_bdev *
spdk_bdev_desc_get_bdev(struct spdk_bdev_desc *desc)
{
assert(desc != NULL);
return desc->bdev;
}
void
spdk_bdev_io_get_iovec(struct spdk_bdev_io *bdev_io, struct iovec **iovp, int *iovcntp)
{
struct iovec *iovs;
int iovcnt;
if (bdev_io == NULL) {
return;
}
switch (bdev_io->type) {
case SPDK_BDEV_IO_TYPE_READ:
case SPDK_BDEV_IO_TYPE_WRITE:
case SPDK_BDEV_IO_TYPE_ZCOPY:
iovs = bdev_io->u.bdev.iovs;
iovcnt = bdev_io->u.bdev.iovcnt;
break;
default:
iovs = NULL;
iovcnt = 0;
break;
}
if (iovp) {
*iovp = iovs;
}
if (iovcntp) {
*iovcntp = iovcnt;
}
}
void *
spdk_bdev_io_get_md_buf(struct spdk_bdev_io *bdev_io)
{
if (bdev_io == NULL) {
return NULL;
}
if (!spdk_bdev_is_md_separate(bdev_io->bdev)) {
return NULL;
}
if (bdev_io->type == SPDK_BDEV_IO_TYPE_READ ||
bdev_io->type == SPDK_BDEV_IO_TYPE_WRITE) {
return bdev_io->u.bdev.md_buf;
}
return NULL;
}
void *
spdk_bdev_io_get_cb_arg(struct spdk_bdev_io *bdev_io)
{
if (bdev_io == NULL) {
assert(false);
return NULL;
}
return bdev_io->internal.caller_ctx;
}
void
spdk_bdev_module_list_add(struct spdk_bdev_module *bdev_module)
{
if (spdk_bdev_module_list_find(bdev_module->name)) {
SPDK_ERRLOG("ERROR: module '%s' already registered.\n", bdev_module->name);
assert(false);
}
/*
* Modules with examine callbacks must be initialized first, so they are
* ready to handle examine callbacks from later modules that will
* register physical bdevs.
*/
if (bdev_module->examine_config != NULL || bdev_module->examine_disk != NULL) {
TAILQ_INSERT_HEAD(&g_bdev_mgr.bdev_modules, bdev_module, internal.tailq);
} else {
TAILQ_INSERT_TAIL(&g_bdev_mgr.bdev_modules, bdev_module, internal.tailq);
}
}
struct spdk_bdev_module *
spdk_bdev_module_list_find(const char *name)
{
struct spdk_bdev_module *bdev_module;
TAILQ_FOREACH(bdev_module, &g_bdev_mgr.bdev_modules, internal.tailq) {
if (strcmp(name, bdev_module->name) == 0) {
break;
}
}
return bdev_module;
}
static void
bdev_write_zero_buffer_next(void *_bdev_io)
{
struct spdk_bdev_io *bdev_io = _bdev_io;
uint64_t num_bytes, num_blocks;
void *md_buf = NULL;
int rc;
num_bytes = spdk_min(_bdev_get_block_size_with_md(bdev_io->bdev) *
bdev_io->u.bdev.split_remaining_num_blocks,
ZERO_BUFFER_SIZE);
num_blocks = num_bytes / _bdev_get_block_size_with_md(bdev_io->bdev);
if (spdk_bdev_is_md_separate(bdev_io->bdev)) {
md_buf = (char *)g_bdev_mgr.zero_buffer +
spdk_bdev_get_block_size(bdev_io->bdev) * num_blocks;
}
rc = bdev_write_blocks_with_md(bdev_io->internal.desc,
spdk_io_channel_from_ctx(bdev_io->internal.ch),
g_bdev_mgr.zero_buffer, md_buf,
bdev_io->u.bdev.split_current_offset_blocks, num_blocks,
bdev_write_zero_buffer_done, bdev_io);
if (rc == 0) {
bdev_io->u.bdev.split_remaining_num_blocks -= num_blocks;
bdev_io->u.bdev.split_current_offset_blocks += num_blocks;
} else if (rc == -ENOMEM) {
bdev_queue_io_wait_with_cb(bdev_io, bdev_write_zero_buffer_next);
} else {
bdev_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
bdev_io->internal.cb(bdev_io, false, bdev_io->internal.caller_ctx);
}
}
static void
bdev_write_zero_buffer_done(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
{
struct spdk_bdev_io *parent_io = cb_arg;
spdk_bdev_free_io(bdev_io);
if (!success) {
parent_io->internal.status = SPDK_BDEV_IO_STATUS_FAILED;
parent_io->internal.cb(parent_io, false, parent_io->internal.caller_ctx);
return;
}
if (parent_io->u.bdev.split_remaining_num_blocks == 0) {
parent_io->internal.status = SPDK_BDEV_IO_STATUS_SUCCESS;
parent_io->internal.cb(parent_io, true, parent_io->internal.caller_ctx);
return;
}
bdev_write_zero_buffer_next(parent_io);
}
static void
bdev_set_qos_limit_done(struct set_qos_limit_ctx *ctx, int status)
{
pthread_mutex_lock(&ctx->bdev->internal.mutex);
ctx->bdev->internal.qos_mod_in_progress = false;
pthread_mutex_unlock(&ctx->bdev->internal.mutex);
if (ctx->cb_fn) {
ctx->cb_fn(ctx->cb_arg, status);
}
free(ctx);
}
static void
bdev_disable_qos_done(void *cb_arg)
{
struct set_qos_limit_ctx *ctx = cb_arg;
struct spdk_bdev *bdev = ctx->bdev;
struct spdk_bdev_io *bdev_io;
struct spdk_bdev_qos *qos;
pthread_mutex_lock(&bdev->internal.mutex);
qos = bdev->internal.qos;
bdev->internal.qos = NULL;
pthread_mutex_unlock(&bdev->internal.mutex);
while (!TAILQ_EMPTY(&qos->queued)) {
/* Send queued I/O back to their original thread for resubmission. */
bdev_io = TAILQ_FIRST(&qos->queued);
TAILQ_REMOVE(&qos->queued, bdev_io, internal.link);
if (bdev_io->internal.io_submit_ch) {
/*
* Channel was changed when sending it to the QoS thread - change it back
* before sending it back to the original thread.
*/
bdev_io->internal.ch = bdev_io->internal.io_submit_ch;
bdev_io->internal.io_submit_ch = NULL;
}
spdk_thread_send_msg(spdk_bdev_io_get_thread(bdev_io),
_bdev_io_submit, bdev_io);
}
if (qos->thread != NULL) {
spdk_put_io_channel(spdk_io_channel_from_ctx(qos->ch));
spdk_poller_unregister(&qos->poller);
}
free(qos);
bdev_set_qos_limit_done(ctx, 0);
}
static void
bdev_disable_qos_msg_done(struct spdk_io_channel_iter *i, int status)
{
void *io_device = spdk_io_channel_iter_get_io_device(i);
struct spdk_bdev *bdev = __bdev_from_io_dev(io_device);
struct set_qos_limit_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
struct spdk_thread *thread;
pthread_mutex_lock(&bdev->internal.mutex);
thread = bdev->internal.qos->thread;
pthread_mutex_unlock(&bdev->internal.mutex);
if (thread != NULL) {
spdk_thread_send_msg(thread, bdev_disable_qos_done, ctx);
} else {
bdev_disable_qos_done(ctx);
}
}
static void
bdev_disable_qos_msg(struct spdk_io_channel_iter *i)
{
struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
struct spdk_bdev_channel *bdev_ch = spdk_io_channel_get_ctx(ch);
bdev_ch->flags &= ~BDEV_CH_QOS_ENABLED;
spdk_for_each_channel_continue(i, 0);
}
static void
bdev_update_qos_rate_limit_msg(void *cb_arg)
{
struct set_qos_limit_ctx *ctx = cb_arg;
struct spdk_bdev *bdev = ctx->bdev;
pthread_mutex_lock(&bdev->internal.mutex);
bdev_qos_update_max_quota_per_timeslice(bdev->internal.qos);
pthread_mutex_unlock(&bdev->internal.mutex);
bdev_set_qos_limit_done(ctx, 0);
}
static void
bdev_enable_qos_msg(struct spdk_io_channel_iter *i)
{
void *io_device = spdk_io_channel_iter_get_io_device(i);
struct spdk_bdev *bdev = __bdev_from_io_dev(io_device);
struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(i);
struct spdk_bdev_channel *bdev_ch = spdk_io_channel_get_ctx(ch);
pthread_mutex_lock(&bdev->internal.mutex);
bdev_enable_qos(bdev, bdev_ch);
pthread_mutex_unlock(&bdev->internal.mutex);
spdk_for_each_channel_continue(i, 0);
}
static void
bdev_enable_qos_done(struct spdk_io_channel_iter *i, int status)
{
struct set_qos_limit_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
bdev_set_qos_limit_done(ctx, status);
}
static void
bdev_set_qos_rate_limits(struct spdk_bdev *bdev, uint64_t *limits)
{
int i;
assert(bdev->internal.qos != NULL);
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
if (limits[i] != SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {
bdev->internal.qos->rate_limits[i].limit = limits[i];
if (limits[i] == 0) {
bdev->internal.qos->rate_limits[i].limit =
SPDK_BDEV_QOS_LIMIT_NOT_DEFINED;
}
}
}
}
void
spdk_bdev_set_qos_rate_limits(struct spdk_bdev *bdev, uint64_t *limits,
void (*cb_fn)(void *cb_arg, int status), void *cb_arg)
{
struct set_qos_limit_ctx *ctx;
uint32_t limit_set_complement;
uint64_t min_limit_per_sec;
int i;
bool disable_rate_limit = true;
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
if (limits[i] == SPDK_BDEV_QOS_LIMIT_NOT_DEFINED) {
continue;
}
if (limits[i] > 0) {
disable_rate_limit = false;
}
if (bdev_qos_is_iops_rate_limit(i) == true) {
min_limit_per_sec = SPDK_BDEV_QOS_MIN_IOS_PER_SEC;
} else {
/* Change from megabyte to byte rate limit */
limits[i] = limits[i] * 1024 * 1024;
min_limit_per_sec = SPDK_BDEV_QOS_MIN_BYTES_PER_SEC;
}
limit_set_complement = limits[i] % min_limit_per_sec;
if (limit_set_complement) {
SPDK_ERRLOG("Requested rate limit %" PRIu64 " is not a multiple of %" PRIu64 "\n",
limits[i], min_limit_per_sec);
limits[i] += min_limit_per_sec - limit_set_complement;
SPDK_ERRLOG("Round up the rate limit to %" PRIu64 "\n", limits[i]);
}
}
ctx = calloc(1, sizeof(*ctx));
if (ctx == NULL) {
cb_fn(cb_arg, -ENOMEM);
return;
}
ctx->cb_fn = cb_fn;
ctx->cb_arg = cb_arg;
ctx->bdev = bdev;
pthread_mutex_lock(&bdev->internal.mutex);
if (bdev->internal.qos_mod_in_progress) {
pthread_mutex_unlock(&bdev->internal.mutex);
free(ctx);
cb_fn(cb_arg, -EAGAIN);
return;
}
bdev->internal.qos_mod_in_progress = true;
if (disable_rate_limit == true && bdev->internal.qos) {
for (i = 0; i < SPDK_BDEV_QOS_NUM_RATE_LIMIT_TYPES; i++) {
if (limits[i] == SPDK_BDEV_QOS_LIMIT_NOT_DEFINED &&
(bdev->internal.qos->rate_limits[i].limit > 0 &&
bdev->internal.qos->rate_limits[i].limit !=
SPDK_BDEV_QOS_LIMIT_NOT_DEFINED)) {
disable_rate_limit = false;
break;
}
}
}
if (disable_rate_limit == false) {
if (bdev->internal.qos == NULL) {
bdev->internal.qos = calloc(1, sizeof(*bdev->internal.qos));
if (!bdev->internal.qos) {
pthread_mutex_unlock(&bdev->internal.mutex);
SPDK_ERRLOG("Unable to allocate memory for QoS tracking\n");
bdev_set_qos_limit_done(ctx, -ENOMEM);
return;
}
}
if (bdev->internal.qos->thread == NULL) {
/* Enabling */
bdev_set_qos_rate_limits(bdev, limits);
spdk_for_each_channel(__bdev_to_io_dev(bdev),
bdev_enable_qos_msg, ctx,
bdev_enable_qos_done);
} else {
/* Updating */
bdev_set_qos_rate_limits(bdev, limits);
spdk_thread_send_msg(bdev->internal.qos->thread,
bdev_update_qos_rate_limit_msg, ctx);
}
} else {
if (bdev->internal.qos != NULL) {
bdev_set_qos_rate_limits(bdev, limits);
/* Disabling */
spdk_for_each_channel(__bdev_to_io_dev(bdev),
bdev_disable_qos_msg, ctx,
bdev_disable_qos_msg_done);
} else {
pthread_mutex_unlock(&bdev->internal.mutex);
bdev_set_qos_limit_done(ctx, 0);
return;
}
}
pthread_mutex_unlock(&bdev->internal.mutex);
}
struct spdk_bdev_histogram_ctx {
spdk_bdev_histogram_status_cb cb_fn;
void *cb_arg;
struct spdk_bdev *bdev;
int status;
};
static void
bdev_histogram_disable_channel_cb(struct spdk_io_channel_iter *i, int status)
{
struct spdk_bdev_histogram_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
pthread_mutex_lock(&ctx->bdev->internal.mutex);
ctx->bdev->internal.histogram_in_progress = false;
pthread_mutex_unlock(&ctx->bdev->internal.mutex);
ctx->cb_fn(ctx->cb_arg, ctx->status);
free(ctx);
}
static void
bdev_histogram_disable_channel(struct spdk_io_channel_iter *i)
{
struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(_ch);
if (ch->histogram != NULL) {
spdk_histogram_data_free(ch->histogram);
ch->histogram = NULL;
}
spdk_for_each_channel_continue(i, 0);
}
static void
bdev_histogram_enable_channel_cb(struct spdk_io_channel_iter *i, int status)
{
struct spdk_bdev_histogram_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
if (status != 0) {
ctx->status = status;
ctx->bdev->internal.histogram_enabled = false;
spdk_for_each_channel(__bdev_to_io_dev(ctx->bdev), bdev_histogram_disable_channel, ctx,
bdev_histogram_disable_channel_cb);
} else {
pthread_mutex_lock(&ctx->bdev->internal.mutex);
ctx->bdev->internal.histogram_in_progress = false;
pthread_mutex_unlock(&ctx->bdev->internal.mutex);
ctx->cb_fn(ctx->cb_arg, ctx->status);
free(ctx);
}
}
static void
bdev_histogram_enable_channel(struct spdk_io_channel_iter *i)
{
struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(_ch);
int status = 0;
if (ch->histogram == NULL) {
ch->histogram = spdk_histogram_data_alloc();
if (ch->histogram == NULL) {
status = -ENOMEM;
}
}
spdk_for_each_channel_continue(i, status);
}
void
spdk_bdev_histogram_enable(struct spdk_bdev *bdev, spdk_bdev_histogram_status_cb cb_fn,
void *cb_arg, bool enable)
{
struct spdk_bdev_histogram_ctx *ctx;
ctx = calloc(1, sizeof(struct spdk_bdev_histogram_ctx));
if (ctx == NULL) {
cb_fn(cb_arg, -ENOMEM);
return;
}
ctx->bdev = bdev;
ctx->status = 0;
ctx->cb_fn = cb_fn;
ctx->cb_arg = cb_arg;
pthread_mutex_lock(&bdev->internal.mutex);
if (bdev->internal.histogram_in_progress) {
pthread_mutex_unlock(&bdev->internal.mutex);
free(ctx);
cb_fn(cb_arg, -EAGAIN);
return;
}
bdev->internal.histogram_in_progress = true;
pthread_mutex_unlock(&bdev->internal.mutex);
bdev->internal.histogram_enabled = enable;
if (enable) {
/* Allocate histogram for each channel */
spdk_for_each_channel(__bdev_to_io_dev(bdev), bdev_histogram_enable_channel, ctx,
bdev_histogram_enable_channel_cb);
} else {
spdk_for_each_channel(__bdev_to_io_dev(bdev), bdev_histogram_disable_channel, ctx,
bdev_histogram_disable_channel_cb);
}
}
struct spdk_bdev_histogram_data_ctx {
spdk_bdev_histogram_data_cb cb_fn;
void *cb_arg;
struct spdk_bdev *bdev;
/** merged histogram data from all channels */
struct spdk_histogram_data *histogram;
};
static void
bdev_histogram_get_channel_cb(struct spdk_io_channel_iter *i, int status)
{
struct spdk_bdev_histogram_data_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
ctx->cb_fn(ctx->cb_arg, status, ctx->histogram);
free(ctx);
}
static void
bdev_histogram_get_channel(struct spdk_io_channel_iter *i)
{
struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(_ch);
struct spdk_bdev_histogram_data_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
int status = 0;
if (ch->histogram == NULL) {
status = -EFAULT;
} else {
spdk_histogram_data_merge(ctx->histogram, ch->histogram);
}
spdk_for_each_channel_continue(i, status);
}
void
spdk_bdev_histogram_get(struct spdk_bdev *bdev, struct spdk_histogram_data *histogram,
spdk_bdev_histogram_data_cb cb_fn,
void *cb_arg)
{
struct spdk_bdev_histogram_data_ctx *ctx;
ctx = calloc(1, sizeof(struct spdk_bdev_histogram_data_ctx));
if (ctx == NULL) {
cb_fn(cb_arg, -ENOMEM, NULL);
return;
}
ctx->bdev = bdev;
ctx->cb_fn = cb_fn;
ctx->cb_arg = cb_arg;
ctx->histogram = histogram;
spdk_for_each_channel(__bdev_to_io_dev(bdev), bdev_histogram_get_channel, ctx,
bdev_histogram_get_channel_cb);
}
size_t
spdk_bdev_get_media_events(struct spdk_bdev_desc *desc, struct spdk_bdev_media_event *events,
size_t max_events)
{
struct media_event_entry *entry;
size_t num_events = 0;
for (; num_events < max_events; ++num_events) {
entry = TAILQ_FIRST(&desc->pending_media_events);
if (entry == NULL) {
break;
}
events[num_events] = entry->event;
TAILQ_REMOVE(&desc->pending_media_events, entry, tailq);
TAILQ_INSERT_TAIL(&desc->free_media_events, entry, tailq);
}
return num_events;
}
int
spdk_bdev_push_media_events(struct spdk_bdev *bdev, const struct spdk_bdev_media_event *events,
size_t num_events)
{
struct spdk_bdev_desc *desc;
struct media_event_entry *entry;
size_t event_id;
int rc = 0;
assert(bdev->media_events);
pthread_mutex_lock(&bdev->internal.mutex);
TAILQ_FOREACH(desc, &bdev->internal.open_descs, link) {
if (desc->write) {
break;
}
}
if (desc == NULL || desc->media_events_buffer == NULL) {
rc = -ENODEV;
goto out;
}
for (event_id = 0; event_id < num_events; ++event_id) {
entry = TAILQ_FIRST(&desc->free_media_events);
if (entry == NULL) {
break;
}
TAILQ_REMOVE(&desc->free_media_events, entry, tailq);
TAILQ_INSERT_TAIL(&desc->pending_media_events, entry, tailq);
entry->event = events[event_id];
}
rc = event_id;
out:
pthread_mutex_unlock(&bdev->internal.mutex);
return rc;
}
void
spdk_bdev_notify_media_management(struct spdk_bdev *bdev)
{
struct spdk_bdev_desc *desc;
pthread_mutex_lock(&bdev->internal.mutex);
TAILQ_FOREACH(desc, &bdev->internal.open_descs, link) {
if (!TAILQ_EMPTY(&desc->pending_media_events)) {
desc->callback.event_fn(SPDK_BDEV_EVENT_MEDIA_MANAGEMENT, bdev,
desc->callback.ctx);
}
}
pthread_mutex_unlock(&bdev->internal.mutex);
}
struct locked_lba_range_ctx {
struct lba_range range;
struct spdk_bdev *bdev;
struct lba_range *current_range;
struct lba_range *owner_range;
struct spdk_poller *poller;
lock_range_cb cb_fn;
void *cb_arg;
};
static void
bdev_lock_error_cleanup_cb(struct spdk_io_channel_iter *i, int status)
{
struct locked_lba_range_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
ctx->cb_fn(ctx->cb_arg, -ENOMEM);
free(ctx);
}
static void
bdev_unlock_lba_range_get_channel(struct spdk_io_channel_iter *i);
static void
bdev_lock_lba_range_cb(struct spdk_io_channel_iter *i, int status)
{
struct locked_lba_range_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
struct spdk_bdev *bdev = ctx->bdev;
if (status == -ENOMEM) {
/* One of the channels could not allocate a range object.
* So we have to go back and clean up any ranges that were
* allocated successfully before we return error status to
* the caller. We can reuse the unlock function to do that
* clean up.
*/
spdk_for_each_channel(__bdev_to_io_dev(bdev),
bdev_unlock_lba_range_get_channel, ctx,
bdev_lock_error_cleanup_cb);
return;
}
/* All channels have locked this range and no I/O overlapping the range
* are outstanding! Set the owner_ch for the range object for the
* locking channel, so that this channel will know that it is allowed
* to write to this range.
*/
ctx->owner_range->owner_ch = ctx->range.owner_ch;
ctx->cb_fn(ctx->cb_arg, status);
/* Don't free the ctx here. Its range is in the bdev's global list of
* locked ranges still, and will be removed and freed when this range
* is later unlocked.
*/
}
static int
bdev_lock_lba_range_check_io(void *_i)
{
struct spdk_io_channel_iter *i = _i;
struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(_ch);
struct locked_lba_range_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
struct lba_range *range = ctx->current_range;
struct spdk_bdev_io *bdev_io;
spdk_poller_unregister(&ctx->poller);
/* The range is now in the locked_ranges, so no new IO can be submitted to this
* range. But we need to wait until any outstanding IO overlapping with this range
* are completed.
*/
TAILQ_FOREACH(bdev_io, &ch->io_submitted, internal.ch_link) {
if (bdev_io_range_is_locked(bdev_io, range)) {
ctx->poller = SPDK_POLLER_REGISTER(bdev_lock_lba_range_check_io, i, 100);
return SPDK_POLLER_BUSY;
}
}
spdk_for_each_channel_continue(i, 0);
return SPDK_POLLER_BUSY;
}
static void
bdev_lock_lba_range_get_channel(struct spdk_io_channel_iter *i)
{
struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(_ch);
struct locked_lba_range_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
struct lba_range *range;
TAILQ_FOREACH(range, &ch->locked_ranges, tailq) {
if (range->length == ctx->range.length &&
range->offset == ctx->range.offset &&
range->locked_ctx == ctx->range.locked_ctx) {
/* This range already exists on this channel, so don't add
* it again. This can happen when a new channel is created
* while the for_each_channel operation is in progress.
* Do not check for outstanding I/O in that case, since the
* range was locked before any I/O could be submitted to the
* new channel.
*/
spdk_for_each_channel_continue(i, 0);
return;
}
}
range = calloc(1, sizeof(*range));
if (range == NULL) {
spdk_for_each_channel_continue(i, -ENOMEM);
return;
}
range->length = ctx->range.length;
range->offset = ctx->range.offset;
range->locked_ctx = ctx->range.locked_ctx;
ctx->current_range = range;
if (ctx->range.owner_ch == ch) {
/* This is the range object for the channel that will hold
* the lock. Store it in the ctx object so that we can easily
* set its owner_ch after the lock is finally acquired.
*/
ctx->owner_range = range;
}
TAILQ_INSERT_TAIL(&ch->locked_ranges, range, tailq);
bdev_lock_lba_range_check_io(i);
}
static void
bdev_lock_lba_range_ctx(struct spdk_bdev *bdev, struct locked_lba_range_ctx *ctx)
{
assert(spdk_get_thread() == spdk_io_channel_get_thread(ctx->range.owner_ch->channel));
/* We will add a copy of this range to each channel now. */
spdk_for_each_channel(__bdev_to_io_dev(bdev), bdev_lock_lba_range_get_channel, ctx,
bdev_lock_lba_range_cb);
}
static bool
bdev_lba_range_overlaps_tailq(struct lba_range *range, lba_range_tailq_t *tailq)
{
struct lba_range *r;
TAILQ_FOREACH(r, tailq, tailq) {
if (bdev_lba_range_overlapped(range, r)) {
return true;
}
}
return false;
}
static int
bdev_lock_lba_range(struct spdk_bdev_desc *desc, struct spdk_io_channel *_ch,
uint64_t offset, uint64_t length,
lock_range_cb cb_fn, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(_ch);
struct locked_lba_range_ctx *ctx;
if (cb_arg == NULL) {
SPDK_ERRLOG("cb_arg must not be NULL\n");
return -EINVAL;
}
ctx = calloc(1, sizeof(*ctx));
if (ctx == NULL) {
return -ENOMEM;
}
ctx->range.offset = offset;
ctx->range.length = length;
ctx->range.owner_ch = ch;
ctx->range.locked_ctx = cb_arg;
ctx->bdev = bdev;
ctx->cb_fn = cb_fn;
ctx->cb_arg = cb_arg;
pthread_mutex_lock(&bdev->internal.mutex);
if (bdev_lba_range_overlaps_tailq(&ctx->range, &bdev->internal.locked_ranges)) {
/* There is an active lock overlapping with this range.
* Put it on the pending list until this range no
* longer overlaps with another.
*/
TAILQ_INSERT_TAIL(&bdev->internal.pending_locked_ranges, &ctx->range, tailq);
} else {
TAILQ_INSERT_TAIL(&bdev->internal.locked_ranges, &ctx->range, tailq);
bdev_lock_lba_range_ctx(bdev, ctx);
}
pthread_mutex_unlock(&bdev->internal.mutex);
return 0;
}
static void
bdev_lock_lba_range_ctx_msg(void *_ctx)
{
struct locked_lba_range_ctx *ctx = _ctx;
bdev_lock_lba_range_ctx(ctx->bdev, ctx);
}
static void
bdev_unlock_lba_range_cb(struct spdk_io_channel_iter *i, int status)
{
struct locked_lba_range_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
struct locked_lba_range_ctx *pending_ctx;
struct spdk_bdev_channel *ch = ctx->range.owner_ch;
struct spdk_bdev *bdev = ch->bdev;
struct lba_range *range, *tmp;
pthread_mutex_lock(&bdev->internal.mutex);
/* Check if there are any pending locked ranges that overlap with this range
* that was just unlocked. If there are, check that it doesn't overlap with any
* other locked ranges before calling bdev_lock_lba_range_ctx which will start
* the lock process.
*/
TAILQ_FOREACH_SAFE(range, &bdev->internal.pending_locked_ranges, tailq, tmp) {
if (bdev_lba_range_overlapped(range, &ctx->range) &&
!bdev_lba_range_overlaps_tailq(range, &bdev->internal.locked_ranges)) {
TAILQ_REMOVE(&bdev->internal.pending_locked_ranges, range, tailq);
pending_ctx = SPDK_CONTAINEROF(range, struct locked_lba_range_ctx, range);
TAILQ_INSERT_TAIL(&bdev->internal.locked_ranges, range, tailq);
spdk_thread_send_msg(spdk_io_channel_get_thread(pending_ctx->range.owner_ch->channel),
bdev_lock_lba_range_ctx_msg, pending_ctx);
}
}
pthread_mutex_unlock(&bdev->internal.mutex);
ctx->cb_fn(ctx->cb_arg, status);
free(ctx);
}
static void
bdev_unlock_lba_range_get_channel(struct spdk_io_channel_iter *i)
{
struct spdk_io_channel *_ch = spdk_io_channel_iter_get_channel(i);
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(_ch);
struct locked_lba_range_ctx *ctx = spdk_io_channel_iter_get_ctx(i);
TAILQ_HEAD(, spdk_bdev_io) io_locked;
struct spdk_bdev_io *bdev_io;
struct lba_range *range;
TAILQ_FOREACH(range, &ch->locked_ranges, tailq) {
if (ctx->range.offset == range->offset &&
ctx->range.length == range->length &&
ctx->range.locked_ctx == range->locked_ctx) {
TAILQ_REMOVE(&ch->locked_ranges, range, tailq);
free(range);
break;
}
}
/* Note: we should almost always be able to assert that the range specified
* was found. But there are some very rare corner cases where a new channel
* gets created simultaneously with a range unlock, where this function
* would execute on that new channel and wouldn't have the range.
* We also use this to clean up range allocations when a later allocation
* fails in the locking path.
* So we can't actually assert() here.
*/
/* Swap the locked IO into a temporary list, and then try to submit them again.
* We could hyper-optimize this to only resubmit locked I/O that overlap
* with the range that was just unlocked, but this isn't a performance path so
* we go for simplicity here.
*/
TAILQ_INIT(&io_locked);
TAILQ_SWAP(&ch->io_locked, &io_locked, spdk_bdev_io, internal.ch_link);
while (!TAILQ_EMPTY(&io_locked)) {
bdev_io = TAILQ_FIRST(&io_locked);
TAILQ_REMOVE(&io_locked, bdev_io, internal.ch_link);
bdev_io_submit(bdev_io);
}
spdk_for_each_channel_continue(i, 0);
}
static int
bdev_unlock_lba_range(struct spdk_bdev_desc *desc, struct spdk_io_channel *_ch,
uint64_t offset, uint64_t length,
lock_range_cb cb_fn, void *cb_arg)
{
struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(desc);
struct spdk_bdev_channel *ch = spdk_io_channel_get_ctx(_ch);
struct locked_lba_range_ctx *ctx;
struct lba_range *range;
bool range_found = false;
/* Let's make sure the specified channel actually has a lock on
* the specified range. Note that the range must match exactly.
*/
TAILQ_FOREACH(range, &ch->locked_ranges, tailq) {
if (range->offset == offset && range->length == length &&
range->owner_ch == ch && range->locked_ctx == cb_arg) {
range_found = true;
break;
}
}
if (!range_found) {
return -EINVAL;
}
pthread_mutex_lock(&bdev->internal.mutex);
/* We confirmed that this channel has locked the specified range. To
* start the unlock the process, we find the range in the bdev's locked_ranges
* and remove it. This ensures new channels don't inherit the locked range.
* Then we will send a message to each channel (including the one specified
* here) to remove the range from its per-channel list.
*/
TAILQ_FOREACH(range, &bdev->internal.locked_ranges, tailq) {
if (range->offset == offset && range->length == length &&
range->locked_ctx == cb_arg) {
break;
}
}
if (range == NULL) {
assert(false);
pthread_mutex_unlock(&bdev->internal.mutex);
return -EINVAL;
}
TAILQ_REMOVE(&bdev->internal.locked_ranges, range, tailq);
ctx = SPDK_CONTAINEROF(range, struct locked_lba_range_ctx, range);
pthread_mutex_unlock(&bdev->internal.mutex);
ctx->cb_fn = cb_fn;
ctx->cb_arg = cb_arg;
spdk_for_each_channel(__bdev_to_io_dev(bdev), bdev_unlock_lba_range_get_channel, ctx,
bdev_unlock_lba_range_cb);
return 0;
}
SPDK_LOG_REGISTER_COMPONENT(bdev)
SPDK_TRACE_REGISTER_FN(bdev_trace, "bdev", TRACE_GROUP_BDEV)
{
spdk_trace_register_owner(OWNER_BDEV, 'b');
spdk_trace_register_object(OBJECT_BDEV_IO, 'i');
spdk_trace_register_description("BDEV_IO_START", TRACE_BDEV_IO_START, OWNER_BDEV,
OBJECT_BDEV_IO, 1,
SPDK_TRACE_ARG_TYPE_INT, "type");
spdk_trace_register_description("BDEV_IO_DONE", TRACE_BDEV_IO_DONE, OWNER_BDEV,
OBJECT_BDEV_IO, 0,
SPDK_TRACE_ARG_TYPE_INT, "");
}
Reference in New Issue View Git Blame Copy Permalink