diff --git a/CMakeLists.txt b/CMakeLists.txt index 6280902..2954d21 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,7 +51,7 @@ add_executable(rat EXCLUDE_FROM_ALL net/rat.cc) target_link_libraries(rat PRIVATE pthread nm ntr gen ${DPDK_LINK_LIBRARIES}) target_compile_options(rat PRIVATE ${CC_FLAGS} ${DPDK_CFLAGS}) -add_executable(birb EXCLUDE_FROM_ALL storage/birb.cc storage/io_gen.cc storage/bdev.cc storage/bdev_thread.cc) +add_executable(birb EXCLUDE_FROM_ALL storage/birb.cc storage/io_gen.cc storage/bdev.cc storage/bdev_thread.cc storage/nvme.cc storage/nvme_thread.cc) target_include_directories(birb PRIVATE ${SPDK_INCLUDE_DIRS} ${DPDK_INCLUDE_DIRS} ${UUID_INCLUDE_DIRS}) target_compile_options(birb PRIVATE ${CC_FLAGS} ${SPDK_CFLAGS} ${UUID_CFLAGS}) target_link_directories(birb PRIVATE ${SPDK_LIBRARY_DIRS} ${SPDK_SYS_STATIC_LIBRARY_DIRS} ${UUID_LIBRARY_DIRS}) diff --git a/inc/storage/driver.hh b/inc/storage/driver.hh index 17ee10e..674d311 100644 --- a/inc/storage/driver.hh +++ b/inc/storage/driver.hh @@ -38,6 +38,7 @@ public: using callback = void (*)(bool, void *); virtual int read(size_t offset, size_t size, char * buffer, callback callback, void * context) = 0; virtual int write(size_t offset, size_t size, char * buffer, callback callback, void * context) = 0; + virtual void poll() = 0; virtual birb_driver::birb_driver_status get_status() = 0; virtual ~birb_driver_thread_context() = default; protected: diff --git a/inc/storage/driver_bdev.hh b/inc/storage/driver_bdev.hh index 6b07220..5408f1f 100644 --- a/inc/storage/driver_bdev.hh +++ b/inc/storage/driver_bdev.hh @@ -38,13 +38,13 @@ public: ~birb_bdev_thread_context() override; int read(size_t offset, size_t size, char * buffer, callback callback, void * context) override; int write(size_t offset, size_t size, char * buffer, callback callback, void * context) override; + void poll() override; birb_driver::birb_driver_status get_status() override; private: struct cb_context { callback cb; void * ctx; - struct spdk_io_channel * ch; }; DISALLOW_EVIL_CONSTRUCTORS(birb_bdev_thread_context); diff --git a/inc/storage/driver_nvme.hh b/inc/storage/driver_nvme.hh new file mode 100644 index 0000000..53957e3 --- /dev/null +++ b/inc/storage/driver_nvme.hh @@ -0,0 +1,64 @@ +#pragma once + +#include "storage/driver.hh" +#include "spdk/nvme.h" +#include "spdk/thread.h" + +class birb_nvme_driver : public birb_driver +{ +public: + birb_nvme_driver(const char * dev_name); + ~birb_nvme_driver() override; + size_t get_capacity() override; + birb_driver_status get_status() override; + birb_driver_type get_type() override; + size_t get_align() override; + + spdk_nvme_ctrlr * get_ctrlr(); + spdk_nvme_ns * get_ns(); + spdk_nvme_io_qpair_opts * get_io_qpair_opts(); + +private: + struct attach_context { + spdk_nvme_ctrlr ** ctrlr; + spdk_nvme_ns ** ns; + const char * dev_name; + }; + + DISALLOW_EVIL_CONSTRUCTORS(birb_nvme_driver); + birb_driver_status status; + spdk_nvme_ctrlr * ctrlr; + spdk_nvme_ns * ns; + spdk_nvme_io_qpair_opts opts; + + static bool probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, struct spdk_nvme_ctrlr_opts *opts); + static void attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, + struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts); +}; + + +class birb_nvme_thread_context : public birb_driver_thread_context +{ +public: + birb_nvme_thread_context(birb_nvme_driver * driver); + ~birb_nvme_thread_context() override; + int read(size_t offset, size_t size, char * buffer, callback callback, void * context) override; + int write(size_t offset, size_t size, char * buffer, callback callback, void * context) override; + void poll() override; + birb_driver::birb_driver_status get_status() override; + +private: + struct cb_context { + callback cb; + void * ctx; + }; + + DISALLOW_EVIL_CONSTRUCTORS(birb_nvme_thread_context); + birb_driver::birb_driver_status status; + birb_nvme_driver * driver; + struct spdk_nvme_qpair * qpair; + + static void io_callback(void *arg, const struct spdk_nvme_cpl *completion); + static uint32_t size_to_lba(size_t size, int lba_size); + static uint64_t addr_to_lba(size_t addr, int lba_size); +}; diff --git a/storage/bdev.cc b/storage/bdev.cc index 646793d..f802d48 100644 --- a/storage/bdev.cc +++ b/storage/bdev.cc @@ -92,4 +92,4 @@ struct spdk_bdev_desc * birb_bdev_driver::get_bdev_desc() { return this->bdev_desc; -} \ No newline at end of file +} diff --git a/storage/bdev_thread.cc b/storage/bdev_thread.cc index 278e33e..19ac0bb 100644 --- a/storage/bdev_thread.cc +++ b/storage/bdev_thread.cc @@ -63,4 +63,10 @@ birb_bdev_thread_context::write(size_t offset, size_t size, char * buffer, callb ctx->cb = callback; ctx->ctx = context; return spdk_bdev_write(driver->get_bdev_desc(), this->io_channel, buffer, offset, size, io_callback, reinterpret_cast(ctx)); +} + +void +birb_bdev_thread_context::poll() +{ + return; } \ No newline at end of file diff --git a/storage/birb.cc b/storage/birb.cc index cb72535..645d868 100644 --- a/storage/birb.cc +++ b/storage/birb.cc @@ -33,6 +33,7 @@ #include "storage/io_gen.hh" #include "storage/driver.hh" #include "storage/driver_bdev.hh" +#include "storage/driver_nvme.hh" static inline uint64_t get_cur_ts_nano() { @@ -228,9 +229,9 @@ static birb_driver * birb_create_driver(const char * driver_name, void * context) { if (strcmp(driver_name, "bdev") == 0) { - return new birb_bdev_driver((const char *)context); + return new birb_bdev_driver(reinterpret_cast(context)); } else if (strcmp(driver_name, "nvme") == 0) { - return nullptr; + return new birb_nvme_driver(reinterpret_cast(context)); } else { return nullptr; } @@ -242,7 +243,7 @@ birb_create_thread_context(birb_driver * driver) if (driver->get_type() == birb_driver::BIRB_DRV_BDEV) { return new birb_bdev_thread_context(dynamic_cast(driver)); } else if (driver->get_type() == birb_driver::BIRB_DRV_NVME) { - return nullptr; + return new birb_nvme_thread_context(dynamic_cast(driver)); } else { return nullptr; } @@ -455,7 +456,8 @@ worker_thread_main(void * arg) while (true) { spdk_thread_poll(spdk_get_thread(), 0, 0); - + driver_thread_ctx->poll(); + if (vars.worker_stop != 0) { if (free_ios.size() >= options.queue_depth) { break; @@ -573,7 +575,7 @@ birb_main(void * arg1 UNUSED) unsigned int total_reqs = 0; unsigned int tid = 0; unsigned long per_thread_cap = 0; - int cur_core = cmask_get_next_cpu(&options.cpumask); + int cur_core; /* initialize driver */ ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "main: initializing device driver for device %s\n", options.dev_name); @@ -600,7 +602,7 @@ birb_main(void * arg1 UNUSED) goto end; } - + cur_core = cmask_get_next_cpu(&options.cpumask); while(cur_core != NEXT_CPU_NULL) { auto * ctx = new struct thread_context; memset(ctx, 0, sizeof(struct thread_context)); @@ -682,19 +684,21 @@ birb_main(void * arg1 UNUSED) // keep stats for (struct thread_context * tctx : worker_threads) { uint64_t last_ts = 0; + uint64_t processed = 0; for (struct io_record * r : *tctx->io_records) { if (r->start_ts >= record_cutoff_time) { if (r->end_ts > last_ts) { last_ts = r->end_ts; } + processed++; output_file << r->end_ts - r->start_ts << std::endl; total_reqs++; } } - ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "main: thread %d processed requests: %zu, last request %lu\n", - tctx->tid, tctx->io_records->size(), last_ts); + ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "main: thread %d processed requests: %lu, last request %lu\n", + tctx->tid, processed, last_ts); } ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "main: total requests: %u, bytes per second: %lu\n", @@ -715,6 +719,7 @@ end: delete tctx; } + exit(0); spdk_app_stop(rc); return; } diff --git a/storage/nvme.cc b/storage/nvme.cc new file mode 100644 index 0000000..65351a8 --- /dev/null +++ b/storage/nvme.cc @@ -0,0 +1,127 @@ +#include +#include "ntr.h" +#include "spdk/nvme.h" +#include "spdk/thread.h" +#include "storage/driver_nvme.hh" + +size_t +birb_nvme_driver::get_capacity() +{ + return spdk_nvme_ns_get_size(this->ns); +} + +birb_driver::birb_driver_status +birb_nvme_driver::get_status() +{ + return this->status; +} + +void +birb_nvme_driver::attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, + struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts UNUSED) +{ + struct spdk_nvme_ns * ns; + auto ctx = reinterpret_cast(cb_ctx); + + ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "birb_nvme_driver: attached to nvme at %s\n", trid->traddr); + + for (int nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); nsid != 0; + nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { + ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); + if (ns == nullptr || !spdk_nvme_ns_is_active(ns)) { + continue; + } + + ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "birb_nvme_driver: namespace id: %d size: %zu LBA size: %u\n", spdk_nvme_ns_get_id(ns), spdk_nvme_ns_get_size(ns), spdk_nvme_ns_get_sector_size(ns)); + /* XXX: use the first namespace */ + break; + } + + *ctx->ns = ns; + *ctx->ctrlr = ctrlr; +} + +bool +birb_nvme_driver::probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, + struct spdk_nvme_ctrlr_opts *opts UNUSED) +{ + printf("birb_nvme_driver: found nvme at %s\n", trid->traddr); + auto ctx = reinterpret_cast(cb_ctx); + + if (strcmp(trid->traddr, ctx->dev_name) == 0) { + return true; + } + return false; +} + +birb_nvme_driver::birb_nvme_driver(const char * dev_name) : status(BIRB_FAIL), + ctrlr(nullptr), + ns(nullptr), + opts() +{ + int rc; + struct spdk_nvme_transport_id trid; + struct attach_context ctx; + ctx.ctrlr = &this->ctrlr; + ctx.ns = &this->ns; + ctx.dev_name = dev_name; + + spdk_nvme_trid_populate_transport(&trid, SPDK_NVME_TRANSPORT_PCIE); + snprintf(trid.subnqn, sizeof(trid.subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); + + rc = spdk_nvme_probe(&trid, reinterpret_cast(&ctx), probe_cb, attach_cb, nullptr); + if (rc != 0) { + ntr(NTR_DEP_USER1, NTR_LEVEL_ERROR, "birb_nvme_driver: failed to probe nvme device: %d\n", rc); + goto end; + } + + if (spdk_nvme_ns_get_csi(this->ns) == SPDK_NVME_CSI_ZNS) { + ntr(NTR_DEP_USER1, NTR_LEVEL_ERROR, "birb_nvme_driver: zoned nvme namespace is unsupported\n"); + spdk_nvme_detach(this->ctrlr); + goto end; + } else { + spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &this->opts, sizeof(this->opts)); + ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "birb_nvme_driver: io queue depth: %d io queue requests: %d\n", opts.io_queue_size, opts.io_queue_requests); + this->status = BIRB_SUCCESS; + } + +end: + return; +} + +birb_nvme_driver::~birb_nvme_driver() +{ + if (this->ctrlr != nullptr) { + spdk_nvme_detach(this->ctrlr); + } +} + +birb_driver::birb_driver_type +birb_nvme_driver::get_type() +{ + return BIRB_DRV_NVME; +} + +size_t +birb_nvme_driver::get_align() +{ + return 0x1000; +} + +spdk_nvme_ctrlr * +birb_nvme_driver::get_ctrlr() +{ + return this->ctrlr; +} + +spdk_nvme_ns * +birb_nvme_driver::get_ns() +{ + return this->ns; +} + +spdk_nvme_io_qpair_opts * +birb_nvme_driver::get_io_qpair_opts() +{ + return &this->opts; +} diff --git a/storage/nvme_thread.cc b/storage/nvme_thread.cc new file mode 100644 index 0000000..ccab86a --- /dev/null +++ b/storage/nvme_thread.cc @@ -0,0 +1,90 @@ +#include + +#include "storage/driver_nvme.hh" +#include "ntr.h" +#include "spdk/bdev.h" +#include "spdk/nvme.h" +#include "spdk/nvme_spec.h" +#include "spdk/thread.h" + +birb_nvme_thread_context::birb_nvme_thread_context(birb_nvme_driver * driver) : status(birb_driver::BIRB_FAIL), + driver(driver), + qpair(nullptr) +{ + struct spdk_nvme_ctrlr * ctrlr = driver->get_ctrlr(); + struct spdk_nvme_qpair * qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, driver->get_io_qpair_opts(), sizeof(struct spdk_nvme_io_qpair_opts)); + if (qpair == nullptr) { + ntr(NTR_DEP_USER1, NTR_LEVEL_ERROR, "birb_nvme_thread_context: could not allocate qpairs.\n"); + } else { + this->qpair = qpair; + status = birb_driver::BIRB_SUCCESS; + } +} + +birb_driver::birb_driver_status +birb_nvme_thread_context::get_status() +{ + return this->status; +} + +birb_nvme_thread_context::~birb_nvme_thread_context() +{ + if (this->qpair != nullptr) { + spdk_nvme_ctrlr_free_io_qpair(this->qpair); + } +} + +/* + * Callback function for io completion. + */ +void +birb_nvme_thread_context::io_callback(void *arg, const struct spdk_nvme_cpl *completion) +{ + bool success = !spdk_nvme_cpl_is_error(completion); + auto ctx = reinterpret_cast(arg); + ctx->cb(success, ctx->ctx); + delete ctx; +} + +uint32_t +birb_nvme_thread_context::size_to_lba(size_t size, int lba_size) +{ + return (size - 1) / lba_size + 1; +} + +uint64_t +birb_nvme_thread_context::addr_to_lba(size_t addr, int lba_size) +{ + return addr / lba_size; +} + +int +birb_nvme_thread_context::read(size_t offset, size_t size, char * buffer, callback callback, void * context) +{ + auto ctx = new struct cb_context; + ctx->cb = callback; + ctx->ctx = context; + + struct spdk_nvme_ns * ns = this->driver->get_ns(); + int lba_size = spdk_nvme_ns_get_sector_size(ns); + return spdk_nvme_ns_cmd_read(ns, this->qpair, buffer, addr_to_lba(offset, lba_size), size_to_lba(size, lba_size), io_callback, reinterpret_cast(ctx), 0); +} + +int +birb_nvme_thread_context::write(size_t offset, size_t size, char * buffer, callback callback, void * context) +{ + auto ctx = new struct cb_context; + ctx->cb = callback; + ctx->ctx = context; + + struct spdk_nvme_ns * ns = this->driver->get_ns(); + int lba_size = spdk_nvme_ns_get_sector_size(ns); + + return spdk_nvme_ns_cmd_write(ns, this->qpair, buffer, addr_to_lba(offset, lba_size), size_to_lba(size, lba_size), io_callback, reinterpret_cast(ctx), 0); +} + +void +birb_nvme_thread_context::poll() +{ + spdk_nvme_qpair_process_completions(this->qpair, 0); +} \ No newline at end of file