d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

vhost: move lcore management to the device backend

Browse Source 
Although Vhost SCSI code is technically capable
of polling different sessions on different lcores,
the underlying SCSI API won't allow allocating
io_channels on more than one lcore.

That's why we will now let device backends assign
lcores by themselves.

The first Vhost SCSI session will now choose one
core from the available ones, and any subsequent
sessions will stick to the same one.

Change-Id: I616cd195a919960dff68508473cea236abf8d6a3
Signed-off-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/c/441581
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Reviewed-by: Changpeng Liu <changpeng.liu@intel.com>
Reviewed-by: Pawel Wodkowski <pawelx.wodkowski@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
This commit is contained in:
Darek Stojaczyk 2019-01-22 16:25:35 +01:00
parent 9dd9adda38
commit 9cddfc0bbf
5 changed files with 98 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lib/vhost/vhost.c
View File

@ -591,7 +591,7 @@ spdk_vhost_session_mem_unregister(struct spdk_vhost_session *vsession)
}
static void
void
spdk_vhost_free_reactor(uint32_t lcore)
{
g_num_ctrlrs[lcore]--;
@ -821,7 +821,7 @@ spdk_vhost_dev_get_cpumask(struct spdk_vhost_dev *vdev)
return vdev->cpumask;
}
static uint32_t
uint32_t
spdk_vhost_allocate_reactor(struct spdk_cpuset *cpumask)
{
uint32_t i, selected_core;
@ -1025,7 +1025,6 @@ stop_device(int vid)
spdk_vhost_session_mem_unregister(vsession);
free(vsession->mem);
spdk_vhost_free_reactor(vsession->lcore);
vsession->lcore = -1;
assert(vdev->active_session_num > 0);
vdev->active_session_num--;
@ -1101,14 +1100,11 @@ start_device(int vid)
}
spdk_vhost_session_set_coalescing(vdev, vsession, NULL);
vsession->lcore = spdk_vhost_allocate_reactor(vdev->cpumask);
spdk_vhost_session_mem_register(vsession);
rc = vdev->backend->start_session(vsession);
if (rc != 0) {
spdk_vhost_session_mem_unregister(vsession);
free(vsession->mem);
spdk_vhost_free_reactor(vsession->lcore);
vsession->lcore = -1;
goto out;
}

27
lib/vhost/vhost_blk.c
View File

@ -719,8 +719,18 @@ out:
static int
spdk_vhost_blk_start(struct spdk_vhost_session *vsession)
{
return spdk_vhost_session_send_event(vsession, spdk_vhost_blk_start_cb,
3, "start session");
int rc;
vsession->lcore = spdk_vhost_allocate_reactor(vsession->vdev->cpumask);
rc = spdk_vhost_session_send_event(vsession, spdk_vhost_blk_start_cb,
3, "start session");
if (rc != 0) {
spdk_vhost_free_reactor(vsession->lcore);
vsession->lcore = -1;
}
return rc;
}
static int
@ -779,8 +789,17 @@ err:
static int
spdk_vhost_blk_stop(struct spdk_vhost_session *vsession)
{
return spdk_vhost_session_send_event(vsession, spdk_vhost_blk_stop_cb,
3, "stop session");
int rc;
rc = spdk_vhost_session_send_event(vsession, spdk_vhost_blk_stop_cb,
3, "stop session");
if (rc != 0) {
return rc;
}
spdk_vhost_free_reactor(vsession->lcore);
vsession->lcore = -1;
return 0;
}
static void

3
lib/vhost/vhost_internal.h
View File

@ -343,6 +343,9 @@ int spdk_vhost_session_send_event(struct spdk_vhost_session *vsession,
*/
void spdk_vhost_session_event_done(void *event_ctx, int response);
void spdk_vhost_free_reactor(uint32_t lcore);
uint32_t spdk_vhost_allocate_reactor(struct spdk_cpuset *cpumask);
void spdk_vhost_lock(void);
void spdk_vhost_unlock(void);
int spdk_remove_vhost_controller(struct spdk_vhost_dev *vdev);

27
lib/vhost/vhost_nvme.c
View File

@ -1106,14 +1106,24 @@ spdk_vhost_nvme_start_cb(struct spdk_vhost_dev *vdev,
static int
spdk_vhost_nvme_start(struct spdk_vhost_session *vsession)
{
int rc;
if (vsession->vdev->active_session_num > 0) {
/* We're trying to start a second session */
SPDK_ERRLOG("Vhost-NVMe devices can support only one simultaneous connection.\n");
return -1;
}
return spdk_vhost_session_send_event(vsession, spdk_vhost_nvme_start_cb,
3, "start session");
vsession->lcore = spdk_vhost_allocate_reactor(vsession->vdev->cpumask);
rc = spdk_vhost_session_send_event(vsession, spdk_vhost_nvme_start_cb,
3, "start session");
if (rc != 0) {
spdk_vhost_free_reactor(vsession->lcore);
vsession->lcore = -1;
}
return rc;
}
static void
@ -1196,8 +1206,17 @@ spdk_vhost_nvme_stop_cb(struct spdk_vhost_dev *vdev,
static int
spdk_vhost_nvme_stop(struct spdk_vhost_session *vsession)
{
return spdk_vhost_session_send_event(vsession, spdk_vhost_nvme_stop_cb,
3, "start session");
int rc;
rc = spdk_vhost_session_send_event(vsession, spdk_vhost_nvme_stop_cb,
3, "start session");
if (rc != 0) {
return rc;
}
spdk_vhost_free_reactor(vsession->lcore);
vsession->lcore = -1;
return 0;
}
static void

62
lib/vhost/vhost_scsi.c
View File

@ -77,6 +77,9 @@ struct spdk_scsi_dev_vhost_state {
struct spdk_vhost_scsi_dev {
struct spdk_vhost_dev vdev;
struct spdk_scsi_dev_vhost_state scsi_dev_state[SPDK_VHOST_SCSI_CTRLR_MAX_DEVS];
/* The CPU chosen to poll I/O of all active vhost sessions */
int32_t lcore;
} __rte_cache_aligned;
struct spdk_vhost_scsi_session {
@ -1217,14 +1220,7 @@ spdk_vhost_scsi_start_cb(struct spdk_vhost_dev *vdev,
int rc;
svsession = to_scsi_session(vsession);
if (svsession == NULL) {
SPDK_ERRLOG("Trying to start non-scsi controller as a scsi one.\n");
rc = -1;
goto out;
}
svdev = to_scsi_dev(vdev);
svsession->svdev = svdev;
svdev = svsession->svdev;
/* validate all I/O queues are in a contiguous index range */
for (i = VIRTIO_SCSI_REQUESTQ; i < vsession->max_queues; i++) {
@ -1266,14 +1262,36 @@ out:
static int
spdk_vhost_scsi_start(struct spdk_vhost_session *vsession)
{
if (vsession->vdev->active_session_num > 0) {
/* We're trying to start a second session */
SPDK_ERRLOG("Vhost-SCSI devices can support only one simultaneous connection.\n");
struct spdk_vhost_scsi_session *svsession;
struct spdk_vhost_scsi_dev *svdev;
int rc;
svsession = to_scsi_session(vsession);
if (svsession == NULL) {
SPDK_ERRLOG("Trying to start non-scsi session as a scsi one.\n");
return -1;
}
return spdk_vhost_session_send_event(vsession, spdk_vhost_scsi_start_cb,
3, "start session");
svdev = to_scsi_dev(vsession->vdev);
svsession->svdev = svdev;
if (svdev->vdev.active_session_num == 0) {
svdev->lcore = spdk_vhost_allocate_reactor(svdev->vdev.cpumask);
}
vsession->lcore = svdev->lcore;
rc = spdk_vhost_session_send_event(vsession, spdk_vhost_scsi_start_cb,
3, "start session");
if (rc != 0) {
vsession->lcore = -1;
if (svdev->vdev.active_session_num == 0) {
spdk_vhost_free_reactor(svdev->lcore);
svdev->lcore = -1;
}
}
return rc;
}
static int
@ -1339,8 +1357,22 @@ err:
static int
spdk_vhost_scsi_stop(struct spdk_vhost_session *vsession)
{
return spdk_vhost_session_send_event(vsession, spdk_vhost_scsi_stop_cb,
3, "stop session");
struct spdk_vhost_scsi_session *svsession;
int rc;
svsession = to_scsi_session(vsession);
rc = spdk_vhost_session_send_event(vsession, spdk_vhost_scsi_stop_cb,
3, "stop session");
if (rc != 0) {
return rc;
}
vsession->lcore = -1;
if (vsession->vdev->active_session_num == 1) {
spdk_vhost_free_reactor(svsession->svdev->lcore);
svsession->svdev->lcore = -1;
}
return 0;
}
static void