d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

bdev: Add an example fio plugin

Browse Source 
Change-Id: I5e0ebb8d452a41ad848b319af9bb978546807d5e
Signed-off-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.gerrithub.io/366495
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Ben Walker 2017-06-06 10:24:07 -07:00 committed by Jim Harris
parent f5a4b0494c
commit 4b8e7d50c0
10 changed files with 856 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -1,5 +1,11 @@
# Changelog # Changelog
## v17.10: (Upcoming Release)
An [fio](http://github.com/axboe/fio) plugin was added that can route
I/O to the bdev layer. See the [plugin documentation](https://github.com/spdk/spdk/blob/master/examples/bdev/fio_plugin/README.md)
for more information.
## v17.07: Build system improvements, userspace vhost-blk target, and GPT bdev ## v17.07: Build system improvements, userspace vhost-blk target, and GPT bdev
### Build System ### Build System

2
examples/Makefile
View File

@ -34,7 +34,7 @@
SPDK_ROOT_DIR := $(abspath $(CURDIR)/..) SPDK_ROOT_DIR := $(abspath $(CURDIR)/..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y += ioat nvme DIRS-y += bdev ioat nvme
.PHONY: all clean $(DIRS-y) .PHONY: all clean $(DIRS-y)

44
examples/bdev/Makefile Normal file
View File

@ -0,0 +1,44 @@
#
# BSD LICENSE
#
# Copyright (c) Intel Corporation.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-$(CONFIG_FIO_PLUGIN) = fio_plugin
.PHONY: all clean $(DIRS-y)
all: $(DIRS-y)
clean: $(DIRS-y)
include $(SPDK_ROOT_DIR)/mk/spdk.subdirs.mk

1
examples/bdev/fio_plugin/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
fio_plugin

59
examples/bdev/fio_plugin/Makefile Normal file
View File

@ -0,0 +1,59 @@
#
# BSD LICENSE
#
# Copyright (c) Intel Corporation.
# Copyright (c) 2015-2016, Micron Technology, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
include $(SPDK_ROOT_DIR)/mk/spdk.app.mk
include $(SPDK_ROOT_DIR)/mk/spdk.modules.mk
APP := fio_plugin
C_SRCS = fio_plugin.c
CFLAGS += -I. $(ENV_CFLAGS) -I$(FIO_SOURCE_DIR)
LDFLAGS += -shared -rdynamic
SPDK_LIB_LIST += util bdev conf copy rpc jsonrpc json log
LIBS += $(BLOCKDEV_MODULES_LINKER_ARGS)
LIBS += $(SPDK_LIB_LINKER_ARGS) $(ENV_LINKER_ARGS)
all: $(APP)
$(APP) : $(OBJS) $(SPDK_LIB_FILES) $(ENV_LIBS) $(BLOCKDEV_MODULES_FILES)
$(LINK_C)
clean:
$(CLEAN_C) $(APP)
include $(SPDK_ROOT_DIR)/mk/spdk.deps.mk

69
examples/bdev/fio_plugin/README.md Normal file
View File

@ -0,0 +1,69 @@
# Compiling fio
Clone the fio source repository from https://github.com/axboe/fio
git clone https://github.com/axboe/fio
Then check out the fio 2.21:
cd fio && git checkout fio-2.21
Finally, compile the code:
make
# Compiling SPDK
Clone the SPDK source repository from https://github.com/spdk/spdk
git clone https://github.com/spdk/spdk
git submodule update --init
Then, run the SPDK configure script to enable fio (point it to the root of the fio repository):
cd spdk
./configure --with-fio=/path/to/fio/repo <other configuration options>
Finally, build SPDK:
make
**Note to advanced users**: These steps assume you're using the DPDK submodule. If you are using your
own version of DPDK, the fio plugin requires that DPDK be compiled with -fPIC. You can compile DPDK
with -fPIC by modifying your DPDK configuration file and adding the line:
EXTRA_CFLAGS=-fPIC
# Usage
To use the SPDK fio plugin with fio, specify the plugin binary using LD_PRELOAD when running
fio and set ioengine=spdk_bdev in the fio configuration file (see example_config.fio in the same
directory as this README).
LD_PRELOAD=<path to spdk repo>/examples/bdev/fio_plugin/fio_plugin fio
The fio configuration file must contain one new parameter:
spdk_conf=./examples/bdev/fio_plugin/bdev.conf
This must point at an SPDK configuration file. There are a number of example configuration
files in the SPDK repository under etc/spdk.
You can specify which block device to run against by setting the filename parameter
to the block device name:
filename=Malloc0
Or for NVMe devices:
filename=Nvme0n1
Currently the SPDK fio plugin is limited to the thread usage model, so fio jobs must also specify thread=1
when using the SPDK fio plugin.
fio also currently has a race condition on shutdown if dynamically loading the ioengine by specifying the
engine's full path via the ioengine parameter - LD_PRELOAD is recommended to avoid this race condition.
When testing random workloads, it is recommended to set norandommap=1. fio's random map
processing consumes extra CPU cycles which will degrade performance over time with
the fio_plugin since all I/O are submitted and completed on a single CPU core.

3
examples/bdev/fio_plugin/bdev.conf.in Normal file
View File

@ -0,0 +1,3 @@
[Malloc]
NumberOfLuns 1
LunSizeInMB 128

16
examples/bdev/fio_plugin/example_config.fio Normal file
View File

@ -0,0 +1,16 @@
[global]
ioengine=spdk_bdev
spdk_conf=./examples/bdev/fio_plugin/bdev.conf.in
thread=1
group_reporting=1
direct=1
verify=0
time_based=1
ramp_time=0
runtime=2
iodepth=128
rw=randrw
bs=4k
[test]
numjobs=1

637
examples/bdev/fio_plugin/fio_plugin.c Normal file
View File

@ -0,0 +1,637 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "spdk/stdinc.h"
#include "spdk/bdev.h"
#include "spdk/copy_engine.h"
#include "spdk/conf.h"
#include "spdk/env.h"
#include "spdk/io_channel.h"
#include "spdk/log.h"
#include "spdk/string.h"
#include "spdk/queue.h"
#include "config-host.h"
#include "fio.h"
#include "optgroup.h"
struct spdk_fio_options {
void *pad;
char *conf;
};
/* Used to pass messages between fio threads */
struct spdk_fio_msg {
spdk_thread_fn cb_fn;
void *cb_arg;
};
/* A polling function */
struct spdk_fio_poller {
spdk_bdev_poller_fn cb_fn;
void *cb_arg;
uint64_t period_microseconds;
TAILQ_ENTRY(spdk_fio_poller) link;
};
struct spdk_fio_request {
struct io_u *io;
struct thread_data *td;
};
struct spdk_fio_target {
struct spdk_bdev *bdev;
struct spdk_bdev_desc *desc;
struct spdk_io_channel *ch;
TAILQ_ENTRY(spdk_fio_target) link;
};
struct spdk_fio_thread {
struct thread_data *td; /* fio thread context */
struct spdk_thread *thread; /* spdk thread context */
struct spdk_ring *ring; /* ring for passing messages to this thread */
TAILQ_HEAD(, spdk_fio_poller) pollers; /* List of registered pollers on this thread */
TAILQ_HEAD(, spdk_fio_target) targets;
struct io_u **iocq; // io completion queue
unsigned int iocq_count; // number of iocq entries filled by last getevents
unsigned int iocq_size; // number of iocq entries allocated
};
static __thread struct spdk_fio_thread *g_thread;
static int spdk_fio_init(struct thread_data *td);
static void spdk_fio_cleanup(struct thread_data *td);
static int spdk_fio_getevents(struct thread_data *td, unsigned int min,
unsigned int max, const struct timespec *t);
static void
spdk_fio_send_msg(spdk_thread_fn fn, void *ctx, void *thread_ctx)
{
struct spdk_fio_thread *thread = thread_ctx;
struct spdk_fio_msg *msg;
size_t count;
msg = calloc(1, sizeof(*msg));
assert(msg != NULL);
msg->cb_fn = fn;
msg->cb_arg = ctx;
count = spdk_ring_enqueue(thread->ring, (void **)&msg, 1);
if (count != 1) {
SPDK_ERRLOG("Unable to send message to thread %p. rc: %lu\n", thread, count);
}
}
static void
spdk_fio_bdev_init_done(void *cb_arg, int rc)
{
*(bool *)cb_arg = true;
}
static void
spdk_fio_start_poller(struct spdk_bdev_poller **ppoller,
spdk_bdev_poller_fn fn,
void *arg,
uint32_t lcore,
uint64_t period_microseconds)
{
struct spdk_fio_thread *fio_thread;
struct spdk_fio_poller *fio_poller;
fio_thread = g_thread;
if (!fio_thread) {
SPDK_ERRLOG("Expected local thread to be initialized, but it was not.\n");
return;
}
fio_poller = calloc(1, sizeof(*fio_poller));
if (!fio_poller) {
SPDK_ERRLOG("Unable to allocate poller\n");
return;
}
fio_poller->cb_fn = fn;
fio_poller->cb_arg = arg;
fio_poller->period_microseconds = period_microseconds;
TAILQ_INSERT_TAIL(&fio_thread->pollers, fio_poller, link);
*ppoller = (struct spdk_bdev_poller *)fio_poller;
}
static void
spdk_fio_stop_poller(struct spdk_bdev_poller **ppoller)
{
struct spdk_fio_poller *fio_poller;
struct spdk_fio_thread *fio_thread;
fio_poller = *(struct spdk_fio_poller **)ppoller;
fio_thread = g_thread;
if (!fio_thread) {
SPDK_ERRLOG("Expected local thread to be initialized, but it was not.\n");
return;
}
TAILQ_REMOVE(&fio_thread->pollers, fio_poller, link);
free(fio_poller);
*ppoller = NULL;
}
static int
spdk_fio_init_thread(struct thread_data *td)
{
struct spdk_fio_thread *fio_thread;
fio_thread = calloc(1, sizeof(*fio_thread));
if (!fio_thread) {
SPDK_ERRLOG("failed to allocate thread local context\n");
return -1;
}
fio_thread->td = td;
td->io_ops_data = fio_thread;
fio_thread->ring = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 128, SPDK_ENV_SOCKET_ID_ANY);
if (!fio_thread->ring) {
SPDK_ERRLOG("failed to allocate ring\n");
free(fio_thread);
return -1;
}
fio_thread->thread = spdk_allocate_thread(spdk_fio_send_msg, fio_thread);
if (!fio_thread->thread) {
spdk_ring_free(fio_thread->ring);
free(fio_thread);
SPDK_ERRLOG("failed to allocate thread\n");
return -1;
}
TAILQ_INIT(&fio_thread->pollers);
fio_thread->iocq_size = td->o.iodepth;
fio_thread->iocq = calloc(fio_thread->iocq_size, sizeof(struct io_u *));
assert(fio_thread->iocq != NULL);
TAILQ_INIT(&fio_thread->targets);
/* Cache the thread in a thread-local variable for easy access */
g_thread = fio_thread;
return 0;
}
static int
spdk_fio_init_env(struct thread_data *td)
{
struct spdk_fio_options *eo;
bool done = false;
int rc;
struct spdk_conf *config;
struct spdk_env_opts opts;
/* Parse the SPDK configuration file */
eo = td->eo;
if (!eo->conf || !strlen(eo->conf)) {
SPDK_ERRLOG("No configuration file provided\n");
return -1;
}
config = spdk_conf_allocate();
if (!config) {
SPDK_ERRLOG("Unable to allocate configuration file\n");
return -1;
}
rc = spdk_conf_read(config, eo->conf);
if (rc != 0) {
SPDK_ERRLOG("Invalid configuration file format\n");
spdk_conf_free(config);
return -1;
}
if (spdk_conf_first_section(config) == NULL) {
SPDK_ERRLOG("Invalid configuration file format");
spdk_conf_free(config);
return -1;
}
spdk_conf_set_as_default(config);
/* Initialize the environment library */
spdk_env_opts_init(&opts);
opts.name = "fio";
spdk_env_init(&opts);
spdk_unaffinitize_thread();
/* Create an SPDK thread temporarily */
rc = spdk_fio_init_thread(td);
if (rc < 0) {
SPDK_ERRLOG("Failed to create initialization thread\n");
return -1;
}
/* Initialize the copy engine */
spdk_copy_engine_initialize();
/* Initialize the bdev layer */
spdk_bdev_initialize(spdk_fio_bdev_init_done, &done,
spdk_fio_start_poller,
spdk_fio_stop_poller);
while (!done) {
spdk_fio_getevents(td, 0, 128, NULL);
}
/* Destroy the temporary SPDK thread */
spdk_fio_cleanup(td);
return 0;
}
/* Called for each thread to fill in the 'real_file_size' member for
* each file associated with this thread. This is called prior to
* the init operation (spdk_fio_init()) below. This call will occur
* on the initial start up thread if 'create_serialize' is true, or
* on the thread actually associated with 'thread_data' if 'create_serialize'
* is false.
*/
static int
spdk_fio_setup(struct thread_data *td)
{
static bool env_initialized = false;
unsigned int i;
struct fio_file *f;
if (!td->o.use_thread) {
SPDK_ERRLOG("must set thread=1 when using spdk plugin\n");
return -1;
}
if (!env_initialized) {
if (spdk_fio_init_env(td)) {
SPDK_ERRLOG("failed to initialize\n");
return -1;
}
env_initialized = true;
}
for_each_file(td, f, i) {
struct spdk_bdev *bdev;
bdev = spdk_bdev_get_by_name(f->file_name);
if (!bdev) {
SPDK_ERRLOG("Unable to find bdev with name %s\n", f->file_name);
return -1;
}
f->real_file_size = spdk_bdev_get_num_blocks(bdev) *
spdk_bdev_get_block_size(bdev);
}
return 0;
}
/* Called for each thread, on that thread, shortly after the thread
* starts.
*/
static int
spdk_fio_init(struct thread_data *td)
{
struct spdk_fio_thread *fio_thread;
unsigned int i;
struct fio_file *f;
int rc;
spdk_fio_init_thread(td);
fio_thread = td->io_ops_data;
for_each_file(td, f, i) {
struct spdk_fio_target *target;
target = calloc(1, sizeof(*target));
if (!target) {
SPDK_ERRLOG("Unable to allocate memory for I/O target.\n");
return -1;
}
target->bdev = spdk_bdev_get_by_name(f->file_name);
if (!target->bdev) {
SPDK_ERRLOG("Unable to find bdev with name %s\n", f->file_name);
free(target);
return -1;
}
rc = spdk_bdev_open(target->bdev, true, NULL, NULL, &target->desc);
if (rc) {
SPDK_ERRLOG("Unable to open bdev %s\n", f->file_name);
free(target);
return -1;
}
target->ch = spdk_bdev_get_io_channel(target->desc);
if (!target->ch) {
SPDK_ERRLOG("Unable to get I/O channel for bdev.\n");
spdk_bdev_close(target->desc);
free(target);
return -1;
}
f->engine_data = target;
TAILQ_INSERT_TAIL(&fio_thread->targets, target, link);
}
return 0;
}
static void
spdk_fio_cleanup(struct thread_data *td)
{
struct spdk_fio_thread *fio_thread = td->io_ops_data;
struct spdk_fio_target *target, *tmp;
g_thread = NULL;
TAILQ_FOREACH_SAFE(target, &fio_thread->targets, link, tmp) {
TAILQ_REMOVE(&fio_thread->targets, target, link);
spdk_put_io_channel(target->ch);
spdk_bdev_close(target->desc);
free(target);
}
spdk_free_thread();
spdk_ring_free(fio_thread->ring);
free(fio_thread);
td->io_ops_data = NULL;
}
static int
spdk_fio_open(struct thread_data *td, struct fio_file *f)
{
return 0;
}
static int
spdk_fio_close(struct thread_data *td, struct fio_file *f)
{
return 0;
}
static int
spdk_fio_iomem_alloc(struct thread_data *td, size_t total_mem)
{
td->orig_buffer = spdk_dma_zmalloc(total_mem, 0x1000, NULL);
return td->orig_buffer == NULL;
}
static void
spdk_fio_iomem_free(struct thread_data *td)
{
spdk_dma_free(td->orig_buffer);
}
static int
spdk_fio_io_u_init(struct thread_data *td, struct io_u *io_u)
{
struct spdk_fio_request *fio_req;
fio_req = calloc(1, sizeof(*fio_req));
if (fio_req == NULL) {
return 1;
}
fio_req->io = io_u;
fio_req->td = td;
io_u->engine_data = fio_req;
return 0;
}
static void
spdk_fio_io_u_free(struct thread_data *td, struct io_u *io_u)
{
struct spdk_fio_request *fio_req = io_u->engine_data;
if (fio_req) {
assert(fio_req->io == io_u);
free(fio_req);
io_u->engine_data = NULL;
}
}
static void
spdk_fio_completion_cb(struct spdk_bdev_io *bdev_io,
bool success,
void *cb_arg)
{
struct spdk_fio_request *fio_req = cb_arg;
struct thread_data *td = fio_req->td;
struct spdk_fio_thread *fio_thread = td->io_ops_data;
assert(fio_thread->iocq_count < fio_thread->iocq_size);
fio_thread->iocq[fio_thread->iocq_count++] = fio_req->io;
spdk_bdev_free_io(bdev_io);
}
static int
spdk_fio_queue(struct thread_data *td, struct io_u *io_u)
{
int rc = 1;
struct spdk_fio_request *fio_req = io_u->engine_data;
struct spdk_fio_target *target = io_u->file->engine_data;
assert(fio_req->td == td);
if (!target) {
SPDK_ERRLOG("Unable to look up correct I/O target.\n");
return -1;
}
switch (io_u->ddir) {
case DDIR_READ:
rc = spdk_bdev_read(target->desc, target->ch,
io_u->buf, io_u->offset, io_u->xfer_buflen,
spdk_fio_completion_cb, fio_req);
break;
case DDIR_WRITE:
rc = spdk_bdev_write(target->desc, target->ch,
io_u->buf, io_u->offset, io_u->xfer_buflen,
spdk_fio_completion_cb, fio_req);
break;
default:
assert(false);
break;
}
if (rc == -ENOMEM) {
return FIO_Q_BUSY;
}
if (rc != 0) {
return -abs(rc);
}
return FIO_Q_QUEUED;
}
static struct io_u *
spdk_fio_event(struct thread_data *td, int event)
{
struct spdk_fio_thread *fio_thread = td->io_ops_data;
assert(event >= 0);
assert((unsigned)event < fio_thread->iocq_count);
return fio_thread->iocq[event];
}
static int
spdk_fio_getevents(struct thread_data *td, unsigned int min,
unsigned int max, const struct timespec *t)
{
struct spdk_fio_thread *fio_thread = td->io_ops_data;
struct timespec t0, t1;
uint64_t timeout = 0;
struct spdk_fio_msg *msg;
struct spdk_fio_poller *p, *tmp;
size_t count;
if (t) {
timeout = t->tv_sec * 1000000000L + t->tv_nsec;
clock_gettime(CLOCK_MONOTONIC_RAW, &t0);
}
fio_thread->iocq_count = 0;
for (;;) {
/* Process new events */
count = spdk_ring_dequeue(fio_thread->ring, (void **)&msg, 1);
if (count > 0) {
msg->cb_fn(msg->cb_arg);
free(msg);
}
/* Call all pollers */
TAILQ_FOREACH_SAFE(p, &fio_thread->pollers, link, tmp) {
p->cb_fn(p->cb_arg);
}
if (fio_thread->iocq_count >= min) {
return fio_thread->iocq_count;
}
if (t) {
clock_gettime(CLOCK_MONOTONIC_RAW, &t1);
uint64_t elapse = ((t1.tv_sec - t0.tv_sec) * 1000000000L)
+ t1.tv_nsec - t0.tv_nsec;
if (elapse > timeout) {
break;
}
}
}
return fio_thread->iocq_count;
}
static int
spdk_fio_invalidate(struct thread_data *td, struct fio_file *f)
{
/* TODO: This should probably send a flush to the device, but for now just return successful. */
return 0;
}
static struct fio_option options[] = {
{
.name = "spdk_conf",
.lname = "SPDK configuration file",
.type = FIO_OPT_STR_STORE,
.off1 = offsetof(struct spdk_fio_options, conf),
.help = "An SPDK configuration file",
.category = FIO_OPT_C_ENGINE,
.group = FIO_OPT_G_INVALID,
},
{
.name = NULL,
},
};
/* FIO imports this structure using dlsym */
struct ioengine_ops ioengine = {
.name = "spdk_bdev",
.version = FIO_IOOPS_VERSION,
.flags = FIO_RAWIO | FIO_NOEXTEND | FIO_NODISKUTIL | FIO_MEMALIGN,
.setup = spdk_fio_setup,
.init = spdk_fio_init,
//.prep = unused,
.queue = spdk_fio_queue,
//.commit = unused,
.getevents = spdk_fio_getevents,
.event = spdk_fio_event,
//.errdetails = unused,
//.cancel = unused,
.cleanup = spdk_fio_cleanup,
.open_file = spdk_fio_open,
.close_file = spdk_fio_close,
.invalidate = spdk_fio_invalidate,
//.unlink_file = unused,
//.get_file_size = unused,
//.terminate = unused,
.iomem_alloc = spdk_fio_iomem_alloc,
.iomem_free = spdk_fio_iomem_free,
.io_u_init = spdk_fio_io_u_init,
.io_u_free = spdk_fio_io_u_free,
.option_struct_size = sizeof(struct spdk_fio_options),
.options = options,
};
static void fio_init spdk_fio_register(void)
{
register_ioengine(&ioengine);
}
static void fio_exit spdk_fio_unregister(void)
{
unregister_ioengine(&ioengine);
}

20
examples/bdev/fio_plugin/full_bench.fio Normal file
View File

@ -0,0 +1,20 @@
[global]
ioengine=spdk_bdev
spdk_conf=./examples/bdev/fio_plugin/bdev.conf.in
thread=1
group_reporting=1
direct=1
verify=0
norandommap=1
cpumask=1
percentile_list=50:99:99.9:99.99:99.999
[4k_randread_qd1]
filename=Malloc0
description="4KiB Random Read QD=1"
bs=4k
rw=randread
iodepth=1
time_based=1
ramp_time=0
runtime=10