bdev/aio: allow user to override block size

Add a file-backed AIO bdev to test it out.

Change-Id: Ifdf206bbdf6cae9379fdc02c80755e96a7198bce
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-on: https://review.gerrithub.io/373673
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>

CHANGELOG.md

@@ -10,6 +10,10 @@ spdk_bdev_unmap() was modified to take an offset and a length in bytes as
 arguments instead of requiring the user to provide an array of SCSI
 unmap descriptors. This limits unmaps to a single contiguous range.
 
+### Linux AIO bdev
+
+The AIO bdev now allows the user to override the auto-detected block size.
+
 ## v17.07: Build system improvements, userspace vhost-blk target, and GPT bdev
 
 ### Build System

doc/bdev.md

@@ -67,11 +67,13 @@ Configuration file syntax:
 
 ~~~
 [AIO]
-  # AIO <file name> <bdev name>
+  # AIO <file name> <bdev name> [<block size>]
   # The file name is the backing device
   # The bdev name can be referenced from elsewhere in the configuration file.
+  # Block size may be omitted to automatically detect the block size of a disk.
   AIO /dev/sdb AIO0
   AIO /dev/sdc AIO1
+  AIO /tmp/myfile AIO2 4096
 ~~~
 
 This exports 2 aio block devices, named AIO0 and AIO1.

etc/spdk/iscsi.conf.in

@@ -135,12 +135,14 @@
 # Users must change this section to match the /dev/sdX devices to be
 # exported as iSCSI LUNs. The devices are accessed using Linux AIO.
 # The format is:
-# AIO <file name> <bdev name>
+# AIO <file name> <bdev name> [<block size>]
 # The file name is the backing device
 # The bdev name can be referenced from elsewhere in the configuration file.
+# Block size may be omitted to automatically detect the block size of a disk.
 [AIO]
   AIO /dev/sdb AIO0
   AIO /dev/sdc AIO1
+  AIO /tmp/myfile AIO2 4096
 
 # The Split virtual block device slices block devices into multiple smaller bdevs.
 [Split]

etc/spdk/nvmf.conf.in

@@ -46,9 +46,11 @@
 # AIO <file name> <bdev name>
 # The file name is the backing device
 # The bdev name can be referenced from elsewhere in the configuration file.
+# Block size may be omitted to automatically detect the block size of a disk.
 [AIO]
   AIO /dev/sdb AIO0
   AIO /dev/sdc AIO1
+  AIO /tmp/myfile AIO2 4096
 
 # Define NVMf protocol global options
 [Nvmf]

lib/bdev/aio/bdev_aio.c

@@ -40,6 +40,7 @@
 #include "spdk/env.h"
 #include "spdk/fd.h"
 #include "spdk/io_channel.h"
+#include "spdk/util.h"
 
 #include "spdk_internal/log.h"
 
@@ -62,11 +63,15 @@ bdev_aio_open(struct file_disk *disk)
 	int fd;
 
 	fd = open(disk->file, O_RDWR | O_DIRECT);
+	if (fd < 0) {
+		/* Try without O_DIRECT for non-disk files */
+		fd = open(disk->file, O_RDWR);
 		if (fd < 0) {
 			perror("open");
 			disk->fd = -1;
 			return -1;
 		}
+	}
 
 	disk->fd = fd;
 
@@ -331,9 +336,10 @@ static void aio_free_disk(struct file_disk *fdisk)
 }
 
 struct spdk_bdev *
-create_aio_disk(const char *name, const char *fname)
+create_aio_disk(const char *name, const char *fname, uint32_t block_size)
 {
 	struct file_disk *fdisk;
+	uint32_t detected_block_size;
 
 	fdisk = calloc(sizeof(*fdisk), 1);
 	if (!fdisk) {
@@ -359,7 +365,46 @@ create_aio_disk(const char *name, const char *fname)
 
 	fdisk->disk.need_aligned_buffer = 1;
 	fdisk->disk.write_cache = 1;
-	fdisk->disk.blocklen = spdk_fd_get_blocklen(fdisk->fd);
+
+	detected_block_size = spdk_fd_get_blocklen(fdisk->fd);
+	if (block_size == 0) {
+		/* User did not specify block size - use autodetected block size. */
+		if (detected_block_size == 0) {
+			SPDK_ERRLOG("Block size could not be auto-detected\n");
+			goto error_return;
+		}
+		block_size = detected_block_size;
+	} else {
+		if (block_size < detected_block_size) {
+			SPDK_ERRLOG("Specified block size %" PRIu32 " is smaller than "
+				    "auto-detected block size %" PRIu32 "\n",
+				    block_size, detected_block_size);
+			goto error_return;
+		} else if (detected_block_size != 0 && block_size != detected_block_size) {
+			SPDK_WARNLOG("Specified block size %" PRIu32 " does not match "
+				     "auto-detected block size %" PRIu32 "\n",
+				     block_size, detected_block_size);
+		}
+	}
+
+	if (block_size < 512) {
+		SPDK_ERRLOG("Invalid block size %" PRIu32 " (must be at least 512).\n", block_size);
+		goto error_return;
+	}
+
+	if (!spdk_u32_is_pow2(block_size)) {
+		SPDK_ERRLOG("Invalid block size %" PRIu32 " (must be a power of 2.)\n", block_size);
+		goto error_return;
+	}
+
+	fdisk->disk.blocklen = block_size;
+
+	if (fdisk->size % fdisk->disk.blocklen != 0) {
+		SPDK_ERRLOG("Disk size %" PRIu64 " is not a multiple of block size %" PRIu32 "\n",
+			    fdisk->size, fdisk->disk.blocklen);
+		goto error_return;
+	}
+
 	fdisk->disk.blockcnt = fdisk->size / fdisk->disk.blocklen;
 	fdisk->disk.ctxt = fdisk;
 
@@ -392,6 +437,8 @@ bdev_aio_initialize(void)
 	while (true) {
 		const char *file;
 		const char *name;
+		const char *block_size_str;
+		uint32_t block_size = 0;
 
 		file = spdk_conf_section_get_nmval(sp, "AIO", i, 0);
 		if (!file) {
@@ -405,7 +452,12 @@ bdev_aio_initialize(void)
 			continue;
 		}
 
-		bdev = create_aio_disk(name, file);
+		block_size_str = spdk_conf_section_get_nmval(sp, "AIO", i, 2);
+		if (block_size_str) {
+			block_size = atoi(block_size_str);
+		}
+
+		bdev = create_aio_disk(name, file, block_size);
 		if (!bdev) {
 			SPDK_ERRLOG("Unable to create AIO bdev from file %s\n", file);
 			i++;

lib/bdev/aio/bdev_aio.h

@@ -67,6 +67,6 @@ struct file_disk {
 	TAILQ_HEAD(, bdev_aio_task) sync_completion_list;
 };
 
-struct spdk_bdev *create_aio_disk(const char *name, const char *fname);
+struct spdk_bdev *create_aio_disk(const char *name, const char *fname, uint32_t block_size);
 
 #endif // SPDK_BDEV_AIO_H

lib/bdev/aio/bdev_aio_rpc.c

@@ -40,6 +40,7 @@
 struct rpc_construct_aio {
 	char *name;
 	char *fname;
+	uint32_t block_size;
 };
 
 static void
@@ -52,6 +53,7 @@ free_rpc_construct_aio(struct rpc_construct_aio *req)
 static const struct spdk_json_object_decoder rpc_construct_aio_decoders[] = {
 	{"name", offsetof(struct rpc_construct_aio, name), spdk_json_decode_string},
 	{"fname", offsetof(struct rpc_construct_aio, fname), spdk_json_decode_string},
+	{"block_size", offsetof(struct rpc_construct_aio, block_size), spdk_json_decode_uint32, true},
 };
 
 static void
@@ -69,7 +71,7 @@ spdk_rpc_construct_aio_bdev(struct spdk_jsonrpc_request *request,
 		goto invalid;
 	}
 
-	bdev = create_aio_disk(req.name, req.fname);
+	bdev = create_aio_disk(req.name, req.fname, req.block_size);
 	if (bdev == NULL) {
 		goto invalid;
 	}

scripts/rpc.py

@@ -190,11 +190,15 @@ def construct_aio_bdev(args):
     params = {'name': args.name,
               'fname': args.fname}
 
+    if args.block_size:
+        params['block_size'] = args.block_size
+
     print_array(jsonrpc_call('construct_aio_bdev', params))
 
 p = subparsers.add_parser('construct_aio_bdev', help='Add a bdev with aio backend')
 p.add_argument('fname', help='Path to device or file (ex: /dev/sda)')
 p.add_argument('name', help='Block device name')
+p.add_argument('block_size', help='Block size for this bdev', type=int, default=argparse.SUPPRESS)
 p.set_defaults(func=construct_aio_bdev)
 
 def construct_nvme_bdev(args):

test/lib/bdev/bdev.conf.in

@@ -12,6 +12,7 @@
 
 [AIO]
   AIO /dev/ram0 AIO0
+  AIO /tmp/aiofile AIO1 2048
 
 [Rpc]
   Enable Yes

test/lib/bdev/blockdev.sh

@@ -10,6 +10,9 @@ source $rootdir/scripts/autotest_common.sh
 
 timing_enter bdev
 
+# Create a file to be used as an AIO backend
+dd if=/dev/zero of=/tmp/aiofile bs=2048 count=5000
+
 cp $testdir/bdev.conf.in $testdir/bdev.conf
 $rootdir/scripts/gen_nvme.sh >> $testdir/bdev.conf
 
@@ -60,5 +63,6 @@ if [ $RUN_NIGHTLY -eq 1 ]; then
 	timing_exit unmap
 fi
 
+rm -f /tmp/aiofile
 rm -f $testdir/bdev.conf
 timing_exit bdev