From 596f92f583cafc02964510a0396f7ad45f8e0e3c Mon Sep 17 00:00:00 2001 From: John Kariuki Date: Thu, 15 Jun 2017 15:38:42 -0700 Subject: [PATCH] fio_plugin: add mem_size to fio config file While doing performance testing for the SPDK NVMe driver using fio with our fio_plugin, I saw the error (transport->ctrlr_create_io_qpair() failed) when running 18 jobs on my system. The error was happening when trying to allocate memory for the trackers at line 890 in the lib/nvme/nvme_pcie.c. Root cause was the fio_plugin.c initializes the environment with only 512 MB of hugepage RAM. I changed opts.mem_size to 1024 and rebuild the plugin the issue was resolved. This patch enables setting the mem_size in the fio config file using parameter named mem_size_mb. E.g. mem_size_mb=1024 Change-Id: I3541b2029a6b36c26f814101313f49c2dd98c9bc Signed-off-by: John Kariuki Reviewed-on: https://review.gerrithub.io/365735 Tested-by: SPDK Automated Test System Reviewed-by: Daniel Verkamp Reviewed-by: Jim Harris --- examples/nvme/fio_plugin/fio_plugin.c | 31 ++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/examples/nvme/fio_plugin/fio_plugin.c b/examples/nvme/fio_plugin/fio_plugin.c index b35ea8fc6a..fde9a2fd8d 100644 --- a/examples/nvme/fio_plugin/fio_plugin.c +++ b/examples/nvme/fio_plugin/fio_plugin.c @@ -45,6 +45,7 @@ #define NVME_IO_ALIGN 4096 static bool spdk_env_initialized; +static int g_mem_size = 512; struct spdk_fio_request { struct io_u *io; @@ -214,7 +215,7 @@ static int spdk_fio_setup(struct thread_data *td) if (!spdk_env_initialized) { spdk_env_opts_init(&opts); opts.name = "fio"; - opts.mem_size = 512; + opts.mem_size = g_mem_size; spdk_env_init(&opts); spdk_env_initialized = true; spdk_unaffinitize_thread(); @@ -467,6 +468,32 @@ static void spdk_fio_cleanup(struct thread_data *td) pthread_mutex_unlock(&mutex); } +static int +str_mem_size_cb(void *data, const char *input) +{ + g_mem_size = atoi(input); + if (!g_mem_size) + g_mem_size = 512; + return 0; +} + +/* This function enables addition of SPDK parameters to the fio config + * Adding new parameters by defining them here and defining a callback + * function to read the parameter value. */ +static struct fio_option options[] = { + { + .name = "mem_size_mb", + .lname = "Memory size in MB", + .type = FIO_OPT_STR_STORE, + .cb = str_mem_size_cb, + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_INVALID, + }, + { + .name = NULL, + }, +}; + /* FIO imports this structure using dlsym */ struct ioengine_ops ioengine = { .name = "spdk", @@ -484,6 +511,8 @@ struct ioengine_ops ioengine = { .io_u_init = spdk_fio_io_u_init, .io_u_free = spdk_fio_io_u_free, .flags = FIO_RAWIO | FIO_NOEXTEND | FIO_NODISKUTIL | FIO_MEMALIGN, + .options = options, + .option_struct_size = 1, }; static void fio_init fio_spdk_register(void)