d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvmf: add save/restore a controller in runtime

Browse Source 
When doing live migration, there are some spdk_nvmf_ctrlr internal
data structures need to be saved/restored.

Change-Id: Ie39482e8c49765c36fc3700fbac4ce47ef306f29
Signed-off-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10058
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
This commit is contained in:
Changpeng Liu 2021-10-29 19:48:33 +08:00 committed by Jim Harris
parent 13f7510f1a
commit 95dd90039f
3 changed files with 73 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
lib/nvmf/ctrlr.c
View File

@ -1922,6 +1922,51 @@ nvmf_ctrlr_save_aers(struct spdk_nvmf_ctrlr *ctrlr, uint16_t *aer_cids,
return ctrlr->nr_aer_reqs;
}
int
nvmf_ctrlr_save_migr_data(struct spdk_nvmf_ctrlr *ctrlr, struct nvmf_ctrlr_migr_data *data)
{
uint32_t num_async_events = 0;
struct spdk_nvmf_async_event_completion *event, *event_tmp;
memcpy(&data->feat, &ctrlr->feat, sizeof(struct spdk_nvmf_ctrlr_feat));
data->cntlid = ctrlr->cntlid;
data->acre_enabled = ctrlr->acre_enabled;
data->notice_aen_mask = ctrlr->notice_aen_mask;
STAILQ_FOREACH_SAFE(event, &ctrlr->async_events, link, event_tmp) {
data->async_events[num_async_events++].raw = event->event.raw;
if (num_async_events == NVMF_MIGR_MAX_PENDING_AERS) {
SPDK_ERRLOG("%p has too many pending AERs\n", ctrlr);
break;
}
}
data->num_async_events = num_async_events;
return 0;
}
int
nvmf_ctrlr_restore_migr_data(struct spdk_nvmf_ctrlr *ctrlr, struct nvmf_ctrlr_migr_data *data)
{
struct spdk_nvmf_async_event_completion *event;
uint32_t i;
memcpy(&ctrlr->feat, &data->feat, sizeof(struct spdk_nvmf_ctrlr_feat));
ctrlr->acre_enabled = data->acre_enabled;
ctrlr->notice_aen_mask = data->notice_aen_mask;
for (i = 0; i < data->num_async_events; i++) {
event = calloc(1, sizeof(struct spdk_nvmf_async_event_completion));
if (!event) {
return -ENOMEM;
}
event->event.raw = data->async_events[i].raw;
STAILQ_INSERT_TAIL(&ctrlr->async_events, event, link);
}
return 0;
}
static int
nvmf_ctrlr_set_features_async_event_configuration(struct spdk_nvmf_request *req)
{

26
lib/nvmf/nvmf_internal.h
View File

@ -282,6 +282,29 @@ struct spdk_nvmf_ctrlr {
TAILQ_ENTRY(spdk_nvmf_ctrlr) link;
};
/* Maximum pending AERs that can be migrated */
#define NVMF_MIGR_MAX_PENDING_AERS 256
/* spdk_nvmf_ctrlr private migration data structure used to save/restore a controller */
struct nvmf_ctrlr_migr_data {
uint32_t opts_size;
uint16_t cntlid;
uint8_t reserved1[2];
struct spdk_nvmf_ctrlr_feat feat;
uint32_t reserved2[2];
uint32_t num_async_events;
uint32_t acre_enabled;
uint64_t notice_aen_mask;
union spdk_nvme_async_event_completion async_events[NVMF_MIGR_MAX_PENDING_AERS];
/* New fields shouldn't go after reserved3 */
uint8_t reserved3[3000];
};
SPDK_STATIC_ASSERT(sizeof(struct nvmf_ctrlr_migr_data) == 0x1000, "Incorrect size");
#define NVMF_MAX_LISTENERS_PER_SUBSYSTEM 16
struct spdk_nvmf_subsystem {
@ -452,6 +475,9 @@ void nvmf_ctrlr_abort_aer(struct spdk_nvmf_ctrlr *ctrlr);
int nvmf_ctrlr_save_aers(struct spdk_nvmf_ctrlr *ctrlr, uint16_t *aer_cids,
uint16_t max_aers);
int nvmf_ctrlr_save_migr_data(struct spdk_nvmf_ctrlr *ctrlr, struct nvmf_ctrlr_migr_data *data);
int nvmf_ctrlr_restore_migr_data(struct spdk_nvmf_ctrlr *ctrlr, struct nvmf_ctrlr_migr_data *data);
/*
* Abort zero-copy requests that already got the buffer (received zcopy_start cb), but haven't
* started zcopy_end. These requests are kept on the outstanding queue, but are not waiting for a

3
lib/nvmf/vfio_user.c
View File

@ -150,7 +150,7 @@ struct nvme_migr_device_state {
uint16_t reserved2[3];
uint16_t nr_aers;
uint16_t aer_cids[256];
uint16_t aer_cids[NVMF_MIGR_MAX_PENDING_AERS];
/* Controller private data offset and length if exist, starting at
* the beginning of this data structure.
@ -390,6 +390,7 @@ vfio_user_migr_data_len(void)
len = NVMF_VFIO_USER_MAX_QPAIRS_PER_CTRLR * (sizeof(struct nvme_migr_sq_state) + sizeof(
struct nvme_migr_cq_state));
len += sizeof(struct nvme_migr_device_state);
len += sizeof(struct nvmf_ctrlr_migr_data);
len += NVME_REG_BAR0_SIZE;
len += NVME_REG_CFG_SIZE;
/* BAR4 */