nvme support

CMakeLists.txt

@@ -51,7 +51,7 @@ add_executable(rat EXCLUDE_FROM_ALL net/rat.cc)
 target_link_libraries(rat PRIVATE pthread nm ntr gen ${DPDK_LINK_LIBRARIES})
 target_compile_options(rat PRIVATE ${CC_FLAGS} ${DPDK_CFLAGS})
 
-add_executable(birb EXCLUDE_FROM_ALL storage/birb.cc storage/io_gen.cc storage/bdev.cc storage/bdev_thread.cc)
+add_executable(birb EXCLUDE_FROM_ALL storage/birb.cc storage/io_gen.cc storage/bdev.cc storage/bdev_thread.cc storage/nvme.cc storage/nvme_thread.cc)
 target_include_directories(birb PRIVATE ${SPDK_INCLUDE_DIRS} ${DPDK_INCLUDE_DIRS} ${UUID_INCLUDE_DIRS})
 target_compile_options(birb PRIVATE ${CC_FLAGS} ${SPDK_CFLAGS} ${UUID_CFLAGS})
 target_link_directories(birb PRIVATE ${SPDK_LIBRARY_DIRS} ${SPDK_SYS_STATIC_LIBRARY_DIRS} ${UUID_LIBRARY_DIRS})

inc/storage/driver.hh

@@ -38,6 +38,7 @@ public:
     using callback = void (*)(bool, void *);
     virtual int read(size_t offset, size_t size, char * buffer, callback callback, void * context) = 0;
     virtual int write(size_t offset, size_t size, char * buffer, callback callback, void * context) = 0;
+    virtual void poll() = 0;
     virtual birb_driver::birb_driver_status get_status() = 0;
     virtual ~birb_driver_thread_context() = default;
 protected:

inc/storage/driver_bdev.hh

@@ -38,13 +38,13 @@ public:
     ~birb_bdev_thread_context() override;
     int read(size_t offset, size_t size, char * buffer, callback callback, void * context) override;
     int write(size_t offset, size_t size, char * buffer, callback callback, void * context) override;
+    void poll() override;
     birb_driver::birb_driver_status get_status() override;
 
 private:
     struct cb_context {
         callback cb;
         void * ctx;
-        struct spdk_io_channel * ch;
     };
 
     DISALLOW_EVIL_CONSTRUCTORS(birb_bdev_thread_context);

inc/storage/driver_nvme.hh

@@ -0,0 +1,64 @@
+#pragma once
+
+#include "storage/driver.hh"
+#include "spdk/nvme.h"
+#include "spdk/thread.h"
+
+class birb_nvme_driver : public birb_driver
+{
+public:
+    birb_nvme_driver(const char * dev_name);
+    ~birb_nvme_driver() override;
+    size_t get_capacity() override;
+    birb_driver_status get_status() override;
+    birb_driver_type get_type() override;
+    size_t get_align() override;
+
+    spdk_nvme_ctrlr * get_ctrlr();
+    spdk_nvme_ns * get_ns();
+    spdk_nvme_io_qpair_opts * get_io_qpair_opts();
+
+private:
+    struct attach_context {
+        spdk_nvme_ctrlr ** ctrlr;
+        spdk_nvme_ns ** ns;
+        const char * dev_name;
+    };
+
+    DISALLOW_EVIL_CONSTRUCTORS(birb_nvme_driver);
+    birb_driver_status status;
+    spdk_nvme_ctrlr * ctrlr;
+    spdk_nvme_ns * ns;
+    spdk_nvme_io_qpair_opts opts;
+
+    static bool probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, struct spdk_nvme_ctrlr_opts *opts);
+    static void attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+	                        struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts);
+};
+
+
+class birb_nvme_thread_context : public birb_driver_thread_context
+{
+public:
+    birb_nvme_thread_context(birb_nvme_driver * driver);
+    ~birb_nvme_thread_context() override;
+    int read(size_t offset, size_t size, char * buffer, callback callback, void * context) override;
+    int write(size_t offset, size_t size, char * buffer, callback callback, void * context) override;
+    void poll() override;
+    birb_driver::birb_driver_status get_status() override;
+
+private:
+    struct cb_context {
+        callback cb;
+        void * ctx;
+    };
+
+    DISALLOW_EVIL_CONSTRUCTORS(birb_nvme_thread_context);
+    birb_driver::birb_driver_status status;
+    birb_nvme_driver * driver;
+    struct spdk_nvme_qpair * qpair;
+
+    static void io_callback(void *arg, const struct spdk_nvme_cpl *completion);
+    static uint32_t size_to_lba(size_t size, int lba_size);
+    static uint64_t addr_to_lba(size_t addr, int lba_size);
+};

storage/bdev_thread.cc

@@ -64,3 +64,9 @@ birb_bdev_thread_context::write(size_t offset, size_t size, char * buffer, callb
 	ctx->ctx = context;
 	return spdk_bdev_write(driver->get_bdev_desc(), this->io_channel, buffer, offset, size, io_callback, reinterpret_cast<void*>(ctx));
 }
+
+void
+birb_bdev_thread_context::poll()
+{
+	return;
+}

storage/birb.cc

@@ -33,6 +33,7 @@
 #include "storage/io_gen.hh"
 #include "storage/driver.hh"
 #include "storage/driver_bdev.hh"
+#include "storage/driver_nvme.hh"
 
 static inline uint64_t get_cur_ts_nano()
 {
@@ -228,9 +229,9 @@ static birb_driver *
 birb_create_driver(const char * driver_name, void * context)
 {
 	if (strcmp(driver_name, "bdev") == 0) {
-		return new birb_bdev_driver((const char *)context);
+		return new birb_bdev_driver(reinterpret_cast<const char *>(context));
 	} else if (strcmp(driver_name, "nvme") == 0) {
-		return nullptr;
+		return new birb_nvme_driver(reinterpret_cast<const char *>(context));
 	} else {
 		return nullptr;
 	}
@@ -242,7 +243,7 @@ birb_create_thread_context(birb_driver * driver)
 	if (driver->get_type() == birb_driver::BIRB_DRV_BDEV) {
 		return new birb_bdev_thread_context(dynamic_cast<birb_bdev_driver *>(driver));
 	} else if (driver->get_type() == birb_driver::BIRB_DRV_NVME) {
-		return nullptr;
+		return new birb_nvme_thread_context(dynamic_cast<birb_nvme_driver *>(driver));
 	} else {
 		return nullptr;
 	}
@@ -455,6 +456,7 @@ worker_thread_main(void * arg)
 	
 	while (true) {
 		spdk_thread_poll(spdk_get_thread(), 0, 0);
+		driver_thread_ctx->poll();
 
 		if (vars.worker_stop != 0) {
 			if (free_ios.size() >= options.queue_depth) {
@@ -573,7 +575,7 @@ birb_main(void * arg1 UNUSED)
 	unsigned int total_reqs = 0;
 	unsigned int tid = 0;
 	unsigned long per_thread_cap = 0;
-	int cur_core = cmask_get_next_cpu(&options.cpumask);
+	int cur_core;
 
 	/* initialize driver */
 	ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "main: initializing device driver for device %s\n", options.dev_name);
@@ -600,7 +602,7 @@ birb_main(void * arg1 UNUSED)
 		goto end;
 	}
 
-
+	cur_core = cmask_get_next_cpu(&options.cpumask);
 	while(cur_core != NEXT_CPU_NULL) {
 		auto * ctx = new struct thread_context;
 		memset(ctx, 0, sizeof(struct thread_context));
@@ -682,19 +684,21 @@ birb_main(void * arg1 UNUSED)
 	// keep stats
 	for (struct thread_context * tctx : worker_threads) {
 		uint64_t last_ts = 0;
+		uint64_t processed = 0;
 		for (struct io_record * r : *tctx->io_records) {
 			if (r->start_ts >= record_cutoff_time) {
 				if (r->end_ts > last_ts) {
 					last_ts = r->end_ts;
 				}
 
+				processed++;
 				output_file << r->end_ts - r->start_ts << std::endl;
 				total_reqs++;
 			}
 		}
 
-		ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "main: thread %d processed requests: %zu, last request %lu\n", 
-											tctx->tid, tctx->io_records->size(), last_ts);
+		ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "main: thread %d processed requests: %lu, last request %lu\n", 
+											tctx->tid, processed, last_ts);
 	}
 
 	ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "main: total requests: %u, bytes per second: %lu\n", 
@@ -715,6 +719,7 @@ end:
 		delete tctx;
 	}
 
+	exit(0);
 	spdk_app_stop(rc);
 	return;
 }

storage/nvme.cc

@@ -0,0 +1,127 @@
+#include <sys/endian.h>
+#include "ntr.h"
+#include "spdk/nvme.h"
+#include "spdk/thread.h"
+#include "storage/driver_nvme.hh"
+
+size_t
+birb_nvme_driver::get_capacity()
+{
+    return spdk_nvme_ns_get_size(this->ns);
+}
+
+birb_driver::birb_driver_status
+birb_nvme_driver::get_status()
+{
+    return this->status;
+}
+
+void
+birb_nvme_driver::attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+	  struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts UNUSED)
+{
+	struct spdk_nvme_ns * ns;
+	auto ctx = reinterpret_cast<struct attach_context *>(cb_ctx);
+
+	ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "birb_nvme_driver: attached to nvme at %s\n", trid->traddr);
+
+	for (int nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); nsid != 0;
+	     nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) {
+		ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid);
+		if (ns == nullptr || !spdk_nvme_ns_is_active(ns)) {
+			continue;
+		}
+
+		ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "birb_nvme_driver: namespace id: %d size: %zu LBA size: %u\n", spdk_nvme_ns_get_id(ns), spdk_nvme_ns_get_size(ns), spdk_nvme_ns_get_sector_size(ns));
+		/* XXX: use the first namespace */
+		break;
+	}
+
+	*ctx->ns = ns;
+	*ctx->ctrlr = ctrlr;
+}
+
+bool
+birb_nvme_driver::probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
+	 struct spdk_nvme_ctrlr_opts *opts UNUSED)
+{
+	printf("birb_nvme_driver: found nvme at %s\n", trid->traddr);
+	auto ctx = reinterpret_cast<struct attach_context *>(cb_ctx);
+
+    if (strcmp(trid->traddr, ctx->dev_name) == 0) {
+		return true;
+    }
+	return false;
+}
+
+birb_nvme_driver::birb_nvme_driver(const char * dev_name) : status(BIRB_FAIL),
+															ctrlr(nullptr),
+															ns(nullptr),
+															opts()
+{
+    int rc;
+    struct spdk_nvme_transport_id trid;
+	struct attach_context ctx;
+	ctx.ctrlr = &this->ctrlr;
+	ctx.ns = &this->ns;
+	ctx.dev_name = dev_name;
+
+    spdk_nvme_trid_populate_transport(&trid, SPDK_NVME_TRANSPORT_PCIE);
+	snprintf(trid.subnqn, sizeof(trid.subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
+
+    rc = spdk_nvme_probe(&trid, reinterpret_cast<void *>(&ctx), probe_cb, attach_cb, nullptr);
+	if (rc != 0) {
+		ntr(NTR_DEP_USER1, NTR_LEVEL_ERROR, "birb_nvme_driver: failed to probe nvme device: %d\n", rc);
+		goto end;
+	}
+
+	if (spdk_nvme_ns_get_csi(this->ns) == SPDK_NVME_CSI_ZNS) {
+		ntr(NTR_DEP_USER1, NTR_LEVEL_ERROR, "birb_nvme_driver: zoned nvme namespace is unsupported\n");
+		spdk_nvme_detach(this->ctrlr);
+		goto end;
+	} else {
+		spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &this->opts, sizeof(this->opts));
+		ntr(NTR_DEP_USER1, NTR_LEVEL_INFO, "birb_nvme_driver: io queue depth: %d io queue requests: %d\n", opts.io_queue_size, opts.io_queue_requests);
+		this->status = BIRB_SUCCESS;
+	}
+
+end:
+	return;	
+}
+
+birb_nvme_driver::~birb_nvme_driver()
+{
+    if (this->ctrlr != nullptr) {
+		spdk_nvme_detach(this->ctrlr);
+	}
+}
+
+birb_driver::birb_driver_type
+birb_nvme_driver::get_type()
+{
+	return BIRB_DRV_NVME;
+}
+
+size_t
+birb_nvme_driver::get_align()
+{
+	return 0x1000;
+}
+
+spdk_nvme_ctrlr * 
+birb_nvme_driver::get_ctrlr()
+{
+	return this->ctrlr;
+}
+
+spdk_nvme_ns * 
+birb_nvme_driver::get_ns()
+{
+	return this->ns;
+}
+
+spdk_nvme_io_qpair_opts * 
+birb_nvme_driver::get_io_qpair_opts()
+{
+	return &this->opts;
+}

storage/nvme_thread.cc

@@ -0,0 +1,90 @@
+#include <sys/endian.h>
+
+#include "storage/driver_nvme.hh"
+#include "ntr.h"
+#include "spdk/bdev.h"
+#include "spdk/nvme.h"
+#include "spdk/nvme_spec.h"
+#include "spdk/thread.h"
+
+birb_nvme_thread_context::birb_nvme_thread_context(birb_nvme_driver * driver) : status(birb_driver::BIRB_FAIL),
+																				driver(driver),
+                                                                                qpair(nullptr)
+{
+	struct spdk_nvme_ctrlr * ctrlr = driver->get_ctrlr();
+    struct spdk_nvme_qpair * qpair = spdk_nvme_ctrlr_alloc_io_qpair(ctrlr, driver->get_io_qpair_opts(), sizeof(struct spdk_nvme_io_qpair_opts));
+    if (qpair == nullptr) {
+        ntr(NTR_DEP_USER1, NTR_LEVEL_ERROR, "birb_nvme_thread_context: could not allocate qpairs.\n");
+    } else {
+        this->qpair = qpair;
+        status = birb_driver::BIRB_SUCCESS;
+    }
+}
+
+birb_driver::birb_driver_status
+birb_nvme_thread_context::get_status()
+{
+    return this->status;
+}
+
+birb_nvme_thread_context::~birb_nvme_thread_context()
+{
+	if (this->qpair != nullptr) {
+		spdk_nvme_ctrlr_free_io_qpair(this->qpair);
+	}
+}
+
+/*
+ * Callback function for io completion.
+ */
+void
+birb_nvme_thread_context::io_callback(void *arg, const struct spdk_nvme_cpl *completion)
+{
+    bool success = !spdk_nvme_cpl_is_error(completion);
+	auto ctx = reinterpret_cast<struct cb_context *>(arg);
+	ctx->cb(success, ctx->ctx);
+	delete ctx;
+}
+
+uint32_t
+birb_nvme_thread_context::size_to_lba(size_t size, int lba_size)
+{
+    return (size - 1) / lba_size + 1;
+}
+
+uint64_t
+birb_nvme_thread_context::addr_to_lba(size_t addr, int lba_size)
+{
+    return addr / lba_size;
+}
+
+int
+birb_nvme_thread_context::read(size_t offset, size_t size, char * buffer, callback callback, void * context)
+{
+	auto ctx = new struct cb_context;
+	ctx->cb = callback;
+	ctx->ctx = context;
+
+    struct spdk_nvme_ns * ns = this->driver->get_ns();
+    int lba_size = spdk_nvme_ns_get_sector_size(ns);
+	return spdk_nvme_ns_cmd_read(ns, this->qpair, buffer, addr_to_lba(offset, lba_size), size_to_lba(size, lba_size), io_callback, reinterpret_cast<void*>(ctx), 0);
+}
+
+int 
+birb_nvme_thread_context::write(size_t offset, size_t size, char * buffer, callback callback, void * context)
+{
+	auto ctx = new struct cb_context;
+	ctx->cb = callback;
+	ctx->ctx = context;
+
+    struct spdk_nvme_ns * ns = this->driver->get_ns();
+    int lba_size = spdk_nvme_ns_get_sector_size(ns);
+
+	return spdk_nvme_ns_cmd_write(ns, this->qpair, buffer, addr_to_lba(offset, lba_size), size_to_lba(size, lba_size), io_callback, reinterpret_cast<void*>(ctx), 0);
+}
+
+void
+birb_nvme_thread_context::poll()
+{
+    spdk_nvme_qpair_process_completions(this->qpair, 0);
+}