d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

module/raid: raid5 module

Browse Source 
Add raid5 module and unit tests. Use './configure' with option
'--with-raid5' to enable it.

Change-Id: I9f07da8c3567fa65499444899c899adaa2e29550
Signed-off-by: Artur Paszkiewicz <artur.paszkiewicz@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/855
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-by: Paul Luse <paul.e.luse@intel.com>
This commit is contained in:
Artur Paszkiewicz 2019-11-04 12:35:50 +01:00 committed by Tomasz Zawadzki
parent b992bb4e46
commit 73763d40ec
12 changed files with 415 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CONFIG
View File

@ -160,3 +160,6 @@ CONFIG_URING_PATH=
# Build with FUSE support
CONFIG_FUSE=n
# Build with RAID5 support
CONFIG_RAID5=n

8
configure vendored
View File

@ -96,6 +96,8 @@ function usage()
echo " No path required."
echo " nvme-cuse Build NVMe driver with support for CUSE-based character devices."
echo " No path required."
echo " raid5 Build with bdev_raid module RAID5 support."
echo " No path required."
echo ""
echo "Environment variables:"
echo ""
@ -409,6 +411,12 @@ for i in "$@"; do
--without-nvme-cuse)
CONFIG[NVME_CUSE]=n
;;
--with-raid5)
CONFIG[RAID5]=y
;;
--without-raid5)
CONFIG[RAID5]=n
;;
--)
break
;;

5
module/bdev/raid/Makefile
View File

@ -36,6 +36,11 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
CFLAGS += -I$(SPDK_ROOT_DIR)/lib/bdev/
C_SRCS = bdev_raid.c bdev_raid_rpc.c raid0.c
ifeq ($(CONFIG_RAID5),y)
C_SRCS += raid5.c
endif
LIBNAME = bdev_raid
include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk

2
module/bdev/raid/bdev_raid.c
View File

@ -846,6 +846,8 @@ static struct {
} g_raid_level_names[] = {
{ "raid0", RAID0 },
{ "0", RAID0 },
{ "raid5", RAID5 },
{ "5", RAID5 },
{ }
};

10
module/bdev/raid/bdev_raid.h
View File

@ -39,6 +39,7 @@
enum raid_level {
INVALID_RAID_LEVEL = -1,
RAID0 = 0,
RAID5 = 5,
};
/*
@ -162,6 +163,9 @@ struct raid_bdev {
/* Module for RAID-level specific operations */
struct raid_bdev_module *module;
/* Private data for the raid module */
void *module_private;
};
#define RAID_FOR_EACH_BASE_BDEV(r, i) \
@ -262,6 +266,12 @@ struct raid_bdev_module {
/* Minimum required number of base bdevs. Must be > 0. */
uint8_t base_bdevs_min;
/*
* Maximum number of base bdevs that can be removed without failing
* the array.
*/
uint8_t base_bdevs_max_degraded;
/*
* Called when the raid is starting, right before changing the state to
* online and registering the bdev. Parameters of the bdev like blockcnt

114
module/bdev/raid/raid5.c Normal file
View File

@ -0,0 +1,114 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "bdev_raid.h"
#include "spdk/env.h"
#include "spdk/io_channel.h"
#include "spdk/string.h"
#include "spdk/util.h"
#include "spdk_internal/log.h"
struct raid5_info {
/* The parent raid bdev */
struct raid_bdev *raid_bdev;
/* Number of data blocks in a stripe (without parity) */
uint64_t stripe_blocks;
/* Number of stripes on this array */
uint64_t total_stripes;
};
static inline uint8_t
raid5_stripe_data_chunks_num(const struct raid_bdev *raid_bdev)
{
return raid_bdev->num_base_bdevs - raid_bdev->module->base_bdevs_max_degraded;
}
static void
raid5_submit_rw_request(struct raid_bdev_io *raid_io)
{
raid_bdev_io_complete(raid_io, SPDK_BDEV_IO_STATUS_FAILED);
}
static int
raid5_start(struct raid_bdev *raid_bdev)
{
uint64_t min_blockcnt = UINT64_MAX;
struct raid_base_bdev_info *base_info;
struct raid5_info *r5info;
r5info = calloc(1, sizeof(*r5info));
if (!r5info) {
SPDK_ERRLOG("Failed to allocate r5info\n");
return -ENOMEM;
}
r5info->raid_bdev = raid_bdev;
RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
min_blockcnt = spdk_min(min_blockcnt, base_info->bdev->blockcnt);
}
r5info->total_stripes = min_blockcnt / raid_bdev->strip_size;
r5info->stripe_blocks = raid_bdev->strip_size * raid5_stripe_data_chunks_num(raid_bdev);
raid_bdev->bdev.blockcnt = r5info->stripe_blocks * r5info->total_stripes;
raid_bdev->bdev.optimal_io_boundary = r5info->stripe_blocks;
raid_bdev->bdev.split_on_optimal_io_boundary = true;
raid_bdev->module_private = r5info;
return 0;
}
static void
raid5_stop(struct raid_bdev *raid_bdev)
{
struct raid5_info *r5info = raid_bdev->module_private;
free(r5info);
}
static struct raid_bdev_module g_raid5_module = {
.level = RAID5,
.base_bdevs_min = 3,
.base_bdevs_max_degraded = 1,
.start = raid5_start,
.stop = raid5_stop,
.submit_rw_request = raid5_submit_rw_request,
};
RAID_MODULE_REGISTER(&g_raid5_module)
SPDK_LOG_REGISTER_COMPONENT("bdev_raid5", SPDK_LOG_BDEV_RAID5)

5
test/common/autotest_common.sh
View File

@ -87,6 +87,7 @@ export RUN_NIGHTLY_FAILING
: ${SPDK_TEST_VMD=0}; export SPDK_TEST_VMD
: ${SPDK_TEST_OPAL=0}; export SPDK_TEST_OPAL
: ${SPDK_AUTOTEST_X=true}; export SPDK_AUTOTEST_X
: ${SPDK_TEST_RAID5=0}; export SPDK_TEST_RAID5
# Export PYTHONPATH with addition of RPC framework. New scripts can be created
# specific use cases for tests.
@ -253,6 +254,10 @@ if [ $SPDK_TEST_BLOBFS -eq 1 ]; then
fi
fi
if [ $SPDK_TEST_RAID5 -eq 1 ]; then
config_params+=' --with-raid5'
fi
# By default, --with-dpdk is not set meaning the SPDK build will use the DPDK submodule.
# If a DPDK installation is found in a well-known location though, WITH_DPDK_DIR will be
# set which will override the default and use that DPDK installation instead.

2
test/unit/lib/bdev/raid/Makefile
View File

@ -36,6 +36,8 @@ include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
DIRS-y = bdev_raid.c
DIRS-$(CONFIG_RAID5) += raid5.c
.PHONY: all clean $(DIRS-y)
all: $(DIRS-y)

1
test/unit/lib/bdev/raid/raid5.c/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
raid5_ut

38
test/unit/lib/bdev/raid/raid5.c/Makefile Normal file
View File

@ -0,0 +1,38 @@
#
# BSD LICENSE
#
# Copyright (c) Intel Corporation.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../../../../..)
TEST_FILE = raid5_ut.c
include $(SPDK_ROOT_DIR)/mk/spdk.unittest.mk

223
test/unit/lib/bdev/raid/raid5.c/raid5_ut.c Normal file
View File

@ -0,0 +1,223 @@
/*-
* BSD LICENSE
*
* Copyright (c) Intel Corporation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE AiRE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include "spdk/stdinc.h"
#include "spdk_cunit.h"
#include "spdk/env.h"
#include "spdk_internal/mock.h"
#include "bdev/raid/raid5.c"
DEFINE_STUB_V(raid_bdev_module_list_add, (struct raid_bdev_module *raid_module));
DEFINE_STUB_V(raid_bdev_io_complete, (struct raid_bdev_io *raid_io,
enum spdk_bdev_io_status status));
struct raid5_params {
uint8_t num_base_bdevs;
uint64_t base_bdev_blockcnt;
uint32_t base_bdev_blocklen;
uint32_t strip_size;
};
static struct raid5_params *g_params;
static size_t g_params_count;
#define ARRAY_FOR_EACH(a, e) \
for (e = a; e < a + SPDK_COUNTOF(a); e++)
#define RAID5_PARAMS_FOR_EACH(p) \
for (p = g_params; p < g_params + g_params_count; p++)
static int
test_setup(void)
{
uint8_t num_base_bdevs_values[] = { 3, 4, 5 };
uint64_t base_bdev_blockcnt_values[] = { 1, 1024, 1024 * 1024 };
uint32_t base_bdev_blocklen_values[] = { 512, 4096 };
uint32_t strip_size_kb_values[] = { 1, 4, 128 };
uint8_t *num_base_bdevs;
uint64_t *base_bdev_blockcnt;
uint32_t *base_bdev_blocklen;
uint32_t *strip_size_kb;
struct raid5_params *params;
g_params_count = SPDK_COUNTOF(num_base_bdevs_values) *
SPDK_COUNTOF(base_bdev_blockcnt_values) *
SPDK_COUNTOF(base_bdev_blocklen_values) *
SPDK_COUNTOF(strip_size_kb_values);
g_params = calloc(g_params_count, sizeof(*g_params));
if (!g_params) {
return -ENOMEM;
}
params = g_params;
ARRAY_FOR_EACH(num_base_bdevs_values, num_base_bdevs) {
ARRAY_FOR_EACH(base_bdev_blockcnt_values, base_bdev_blockcnt) {
ARRAY_FOR_EACH(base_bdev_blocklen_values, base_bdev_blocklen) {
ARRAY_FOR_EACH(strip_size_kb_values, strip_size_kb) {
params->num_base_bdevs = *num_base_bdevs;
params->base_bdev_blockcnt = *base_bdev_blockcnt;
params->base_bdev_blocklen = *base_bdev_blocklen;
params->strip_size = *strip_size_kb * 1024 / *base_bdev_blocklen;
if (params->strip_size == 0 ||
params->strip_size > *base_bdev_blockcnt) {
g_params_count--;
continue;
}
params++;
}
}
}
}
return 0;
}
static int
test_cleanup(void)
{
free(g_params);
return 0;
}
static struct raid_bdev *
create_raid_bdev(struct raid5_params *params)
{
struct raid_bdev *raid_bdev;
struct raid_base_bdev_info *base_info;
raid_bdev = calloc(1, sizeof(*raid_bdev));
SPDK_CU_ASSERT_FATAL(raid_bdev != NULL);
raid_bdev->module = &g_raid5_module;
raid_bdev->num_base_bdevs = params->num_base_bdevs;
raid_bdev->base_bdev_info = calloc(raid_bdev->num_base_bdevs,
sizeof(struct raid_base_bdev_info));
SPDK_CU_ASSERT_FATAL(raid_bdev->base_bdev_info != NULL);
RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
base_info->bdev = calloc(1, sizeof(*base_info->bdev));
SPDK_CU_ASSERT_FATAL(base_info->bdev != NULL);
base_info->bdev->blockcnt = params->base_bdev_blockcnt;
base_info->bdev->blocklen = params->base_bdev_blocklen;
}
raid_bdev->strip_size = params->strip_size;
raid_bdev->strip_size_shift = spdk_u32log2(raid_bdev->strip_size);
raid_bdev->bdev.blocklen = params->base_bdev_blocklen;
return raid_bdev;
}
static void
delete_raid_bdev(struct raid_bdev *raid_bdev)
{
struct raid_base_bdev_info *base_info;
RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) {
free(base_info->bdev);
}
free(raid_bdev->base_bdev_info);
free(raid_bdev);
}
static struct raid5_info *
create_raid5(struct raid5_params *params)
{
struct raid_bdev *raid_bdev = create_raid_bdev(params);
SPDK_CU_ASSERT_FATAL(raid5_start(raid_bdev) == 0);
return raid_bdev->module_private;
}
static void
delete_raid5(struct raid5_info *r5info)
{
struct raid_bdev *raid_bdev = r5info->raid_bdev;
raid5_stop(raid_bdev);
delete_raid_bdev(raid_bdev);
}
static void
test_raid5_start(void)
{
struct raid5_params *params;
RAID5_PARAMS_FOR_EACH(params) {
struct raid5_info *r5info;
r5info = create_raid5(params);
CU_ASSERT_EQUAL(r5info->stripe_blocks, params->strip_size * (params->num_base_bdevs - 1));
CU_ASSERT_EQUAL(r5info->total_stripes, params->base_bdev_blockcnt / params->strip_size);
CU_ASSERT_EQUAL(r5info->raid_bdev->bdev.blockcnt,
(params->base_bdev_blockcnt - params->base_bdev_blockcnt % params->strip_size) *
(params->num_base_bdevs - 1));
CU_ASSERT_EQUAL(r5info->raid_bdev->bdev.optimal_io_boundary, r5info->stripe_blocks);
delete_raid5(r5info);
}
}
int
main(int argc, char **argv)
{
CU_pSuite suite = NULL;
unsigned int num_failures;
if (CU_initialize_registry() != CUE_SUCCESS) {
return CU_get_error();
}
suite = CU_add_suite("raid5", test_setup, test_cleanup);
if (suite == NULL) {
CU_cleanup_registry();
return CU_get_error();
}
if (CU_add_test(suite, "test_raid5_start", test_raid5_start) == NULL) {
CU_cleanup_registry();
return CU_get_error();
}
CU_basic_set_mode(CU_BRM_VERBOSE);
CU_basic_run_tests();
num_failures = CU_get_number_of_failures();
CU_cleanup_registry();
return num_failures;
}

4
test/unit/unittest.sh
View File

@ -168,6 +168,10 @@ if [ $SPDK_TEST_PMDK -eq 1 ]; then
run_test "unittest_bdev_pmem" $valgrind $testdir/lib/bdev/pmem/bdev_pmem_ut
fi
if [ $SPDK_TEST_RAID5 -eq 1 ]; then
run_test "unittest_bdev_raid5" $valgrind $testdir/lib/bdev/raid/raid5.c/raid5_ut
fi
run_test "unittest_blob_blobfs" unittest_blob
run_test "unittest_event" unittest_event
if [ $(uname -s) = Linux ]; then