From 671179409cef93aafbd8cb88ad6e7c054da5416e Mon Sep 17 00:00:00 2001 From: Daniel Verkamp Date: Wed, 9 Aug 2017 17:28:32 -0700 Subject: [PATCH] bdev/aio: allow user to override block size Add a file-backed AIO bdev to test it out. Change-Id: Ifdf206bbdf6cae9379fdc02c80755e96a7198bce Signed-off-by: Daniel Verkamp Reviewed-on: https://review.gerrithub.io/373673 Tested-by: SPDK Automated Test System Reviewed-by: Ben Walker Reviewed-by: Changpeng Liu Reviewed-by: Jim Harris --- CHANGELOG.md | 4 +++ doc/bdev.md | 4 ++- etc/spdk/iscsi.conf.in | 4 ++- etc/spdk/nvmf.conf.in | 2 ++ lib/bdev/aio/bdev_aio.c | 64 +++++++++++++++++++++++++++++++++---- lib/bdev/aio/bdev_aio.h | 2 +- lib/bdev/aio/bdev_aio_rpc.c | 4 ++- scripts/rpc.py | 4 +++ test/lib/bdev/bdev.conf.in | 1 + test/lib/bdev/blockdev.sh | 4 +++ 10 files changed, 83 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index abed4e1a74..c6b3766e92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ spdk_bdev_unmap() was modified to take an offset and a length in bytes as arguments instead of requiring the user to provide an array of SCSI unmap descriptors. This limits unmaps to a single contiguous range. +### Linux AIO bdev + +The AIO bdev now allows the user to override the auto-detected block size. + ## v17.07: Build system improvements, userspace vhost-blk target, and GPT bdev ### Build System diff --git a/doc/bdev.md b/doc/bdev.md index 820a0592d3..6b285b4498 100644 --- a/doc/bdev.md +++ b/doc/bdev.md @@ -67,11 +67,13 @@ Configuration file syntax: ~~~ [AIO] - # AIO + # AIO [] # The file name is the backing device # The bdev name can be referenced from elsewhere in the configuration file. + # Block size may be omitted to automatically detect the block size of a disk. AIO /dev/sdb AIO0 AIO /dev/sdc AIO1 + AIO /tmp/myfile AIO2 4096 ~~~ This exports 2 aio block devices, named AIO0 and AIO1. diff --git a/etc/spdk/iscsi.conf.in b/etc/spdk/iscsi.conf.in index 8ee0445541..19375de48f 100644 --- a/etc/spdk/iscsi.conf.in +++ b/etc/spdk/iscsi.conf.in @@ -135,12 +135,14 @@ # Users must change this section to match the /dev/sdX devices to be # exported as iSCSI LUNs. The devices are accessed using Linux AIO. # The format is: -# AIO +# AIO [] # The file name is the backing device # The bdev name can be referenced from elsewhere in the configuration file. +# Block size may be omitted to automatically detect the block size of a disk. [AIO] AIO /dev/sdb AIO0 AIO /dev/sdc AIO1 + AIO /tmp/myfile AIO2 4096 # The Split virtual block device slices block devices into multiple smaller bdevs. [Split] diff --git a/etc/spdk/nvmf.conf.in b/etc/spdk/nvmf.conf.in index 949234534e..febd2cf035 100644 --- a/etc/spdk/nvmf.conf.in +++ b/etc/spdk/nvmf.conf.in @@ -46,9 +46,11 @@ # AIO # The file name is the backing device # The bdev name can be referenced from elsewhere in the configuration file. +# Block size may be omitted to automatically detect the block size of a disk. [AIO] AIO /dev/sdb AIO0 AIO /dev/sdc AIO1 + AIO /tmp/myfile AIO2 4096 # Define NVMf protocol global options [Nvmf] diff --git a/lib/bdev/aio/bdev_aio.c b/lib/bdev/aio/bdev_aio.c index 787649659c..3c05bac542 100644 --- a/lib/bdev/aio/bdev_aio.c +++ b/lib/bdev/aio/bdev_aio.c @@ -40,6 +40,7 @@ #include "spdk/env.h" #include "spdk/fd.h" #include "spdk/io_channel.h" +#include "spdk/util.h" #include "spdk_internal/log.h" @@ -63,9 +64,13 @@ bdev_aio_open(struct file_disk *disk) fd = open(disk->file, O_RDWR | O_DIRECT); if (fd < 0) { - perror("open"); - disk->fd = -1; - return -1; + /* Try without O_DIRECT for non-disk files */ + fd = open(disk->file, O_RDWR); + if (fd < 0) { + perror("open"); + disk->fd = -1; + return -1; + } } disk->fd = fd; @@ -331,9 +336,10 @@ static void aio_free_disk(struct file_disk *fdisk) } struct spdk_bdev * -create_aio_disk(const char *name, const char *fname) +create_aio_disk(const char *name, const char *fname, uint32_t block_size) { struct file_disk *fdisk; + uint32_t detected_block_size; fdisk = calloc(sizeof(*fdisk), 1); if (!fdisk) { @@ -359,7 +365,46 @@ create_aio_disk(const char *name, const char *fname) fdisk->disk.need_aligned_buffer = 1; fdisk->disk.write_cache = 1; - fdisk->disk.blocklen = spdk_fd_get_blocklen(fdisk->fd); + + detected_block_size = spdk_fd_get_blocklen(fdisk->fd); + if (block_size == 0) { + /* User did not specify block size - use autodetected block size. */ + if (detected_block_size == 0) { + SPDK_ERRLOG("Block size could not be auto-detected\n"); + goto error_return; + } + block_size = detected_block_size; + } else { + if (block_size < detected_block_size) { + SPDK_ERRLOG("Specified block size %" PRIu32 " is smaller than " + "auto-detected block size %" PRIu32 "\n", + block_size, detected_block_size); + goto error_return; + } else if (detected_block_size != 0 && block_size != detected_block_size) { + SPDK_WARNLOG("Specified block size %" PRIu32 " does not match " + "auto-detected block size %" PRIu32 "\n", + block_size, detected_block_size); + } + } + + if (block_size < 512) { + SPDK_ERRLOG("Invalid block size %" PRIu32 " (must be at least 512).\n", block_size); + goto error_return; + } + + if (!spdk_u32_is_pow2(block_size)) { + SPDK_ERRLOG("Invalid block size %" PRIu32 " (must be a power of 2.)\n", block_size); + goto error_return; + } + + fdisk->disk.blocklen = block_size; + + if (fdisk->size % fdisk->disk.blocklen != 0) { + SPDK_ERRLOG("Disk size %" PRIu64 " is not a multiple of block size %" PRIu32 "\n", + fdisk->size, fdisk->disk.blocklen); + goto error_return; + } + fdisk->disk.blockcnt = fdisk->size / fdisk->disk.blocklen; fdisk->disk.ctxt = fdisk; @@ -392,6 +437,8 @@ bdev_aio_initialize(void) while (true) { const char *file; const char *name; + const char *block_size_str; + uint32_t block_size = 0; file = spdk_conf_section_get_nmval(sp, "AIO", i, 0); if (!file) { @@ -405,7 +452,12 @@ bdev_aio_initialize(void) continue; } - bdev = create_aio_disk(name, file); + block_size_str = spdk_conf_section_get_nmval(sp, "AIO", i, 2); + if (block_size_str) { + block_size = atoi(block_size_str); + } + + bdev = create_aio_disk(name, file, block_size); if (!bdev) { SPDK_ERRLOG("Unable to create AIO bdev from file %s\n", file); i++; diff --git a/lib/bdev/aio/bdev_aio.h b/lib/bdev/aio/bdev_aio.h index 8056c11409..8831befc6a 100644 --- a/lib/bdev/aio/bdev_aio.h +++ b/lib/bdev/aio/bdev_aio.h @@ -67,6 +67,6 @@ struct file_disk { TAILQ_HEAD(, bdev_aio_task) sync_completion_list; }; -struct spdk_bdev *create_aio_disk(const char *name, const char *fname); +struct spdk_bdev *create_aio_disk(const char *name, const char *fname, uint32_t block_size); #endif // SPDK_BDEV_AIO_H diff --git a/lib/bdev/aio/bdev_aio_rpc.c b/lib/bdev/aio/bdev_aio_rpc.c index 9f59d7fe36..609164d2a0 100644 --- a/lib/bdev/aio/bdev_aio_rpc.c +++ b/lib/bdev/aio/bdev_aio_rpc.c @@ -40,6 +40,7 @@ struct rpc_construct_aio { char *name; char *fname; + uint32_t block_size; }; static void @@ -52,6 +53,7 @@ free_rpc_construct_aio(struct rpc_construct_aio *req) static const struct spdk_json_object_decoder rpc_construct_aio_decoders[] = { {"name", offsetof(struct rpc_construct_aio, name), spdk_json_decode_string}, {"fname", offsetof(struct rpc_construct_aio, fname), spdk_json_decode_string}, + {"block_size", offsetof(struct rpc_construct_aio, block_size), spdk_json_decode_uint32, true}, }; static void @@ -69,7 +71,7 @@ spdk_rpc_construct_aio_bdev(struct spdk_jsonrpc_request *request, goto invalid; } - bdev = create_aio_disk(req.name, req.fname); + bdev = create_aio_disk(req.name, req.fname, req.block_size); if (bdev == NULL) { goto invalid; } diff --git a/scripts/rpc.py b/scripts/rpc.py index 006ea0a1f3..6fb332fbad 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -190,11 +190,15 @@ def construct_aio_bdev(args): params = {'name': args.name, 'fname': args.fname} + if args.block_size: + params['block_size'] = args.block_size + print_array(jsonrpc_call('construct_aio_bdev', params)) p = subparsers.add_parser('construct_aio_bdev', help='Add a bdev with aio backend') p.add_argument('fname', help='Path to device or file (ex: /dev/sda)') p.add_argument('name', help='Block device name') +p.add_argument('block_size', help='Block size for this bdev', type=int, default=argparse.SUPPRESS) p.set_defaults(func=construct_aio_bdev) def construct_nvme_bdev(args): diff --git a/test/lib/bdev/bdev.conf.in b/test/lib/bdev/bdev.conf.in index 278d0e642b..2e900418b9 100644 --- a/test/lib/bdev/bdev.conf.in +++ b/test/lib/bdev/bdev.conf.in @@ -12,6 +12,7 @@ [AIO] AIO /dev/ram0 AIO0 + AIO /tmp/aiofile AIO1 2048 [Rpc] Enable Yes diff --git a/test/lib/bdev/blockdev.sh b/test/lib/bdev/blockdev.sh index c24d80c72e..6a7a885157 100755 --- a/test/lib/bdev/blockdev.sh +++ b/test/lib/bdev/blockdev.sh @@ -10,6 +10,9 @@ source $rootdir/scripts/autotest_common.sh timing_enter bdev +# Create a file to be used as an AIO backend +dd if=/dev/zero of=/tmp/aiofile bs=2048 count=5000 + cp $testdir/bdev.conf.in $testdir/bdev.conf $rootdir/scripts/gen_nvme.sh >> $testdir/bdev.conf @@ -60,5 +63,6 @@ if [ $RUN_NIGHTLY -eq 1 ]; then timing_exit unmap fi +rm -f /tmp/aiofile rm -f $testdir/bdev.conf timing_exit bdev