d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

vhost: introduce per-session coalescing settings

Browse Source 
Keep all coalescing variables inside the session struct.
Interrupt coalescing is still configured with the device-
pecific APIs, but those will now transparently propagate
the change to all active connections.

This is the last piece that held struct spdk_vhost_dev
tied with the session's lcore. Now that device
settings aren't actively polled by any sessions, they
only need to be synchronized with the global vhost lock.

This will potentially let us get rid of the vhost external
events API, allowing user to lock the mutex directly,
set coalescing params directly, and transparently let
the internal spdk_vhost_dev_foreach_session() do the
tricky synchronization.

Change-Id: Ifba96d241c736d33376861fa894c738e7d9b5b40
Signed-off-by: Darek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/c/437777
Reviewed-by: Pawel Wodkowski <pawelx.wodkowski@intel.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
This commit is contained in:
Darek Stojaczyk 2018-12-17 03:45:35 +01:00 committed by Jim Harris
parent 1e18d9cd9d
commit ca3962bf4a
2 changed files with 38 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
lib/vhost/vhost.c
View File

@ -268,19 +268,18 @@ spdk_vhost_vq_used_signal(struct spdk_vhost_session *vsession,
static void
check_session_io_stats(struct spdk_vhost_session *vsession, uint64_t now)
{
struct spdk_vhost_dev *vdev = vsession->vdev;
struct spdk_vhost_virtqueue *virtqueue;
uint32_t irq_delay_base = vdev->coalescing_delay_time_base;
uint32_t io_threshold = vdev->coalescing_io_rate_threshold;
uint32_t irq_delay_base = vsession->coalescing_delay_time_base;
uint32_t io_threshold = vsession->coalescing_io_rate_threshold;
int32_t irq_delay;
uint32_t req_cnt;
uint16_t q_idx;
if (now < vdev->next_stats_check_time) {
if (now < vsession->next_stats_check_time) {
return;
}
vdev->next_stats_check_time = now + vdev->stats_check_interval;
vsession->next_stats_check_time = now + vsession->stats_check_interval;
for (q_idx = 0; q_idx < vsession->max_queues; q_idx++) {
virtqueue = &vsession->virtqueue[q_idx];
@ -300,12 +299,11 @@ check_session_io_stats(struct spdk_vhost_session *vsession, uint64_t now)
void
spdk_vhost_session_used_signal(struct spdk_vhost_session *vsession)
{
struct spdk_vhost_dev *vdev = vsession->vdev;
struct spdk_vhost_virtqueue *virtqueue;
uint64_t now;
uint16_t q_idx;
if (vdev->coalescing_delay_time_base == 0) {
if (vsession->coalescing_delay_time_base == 0) {
for (q_idx = 0; q_idx < vsession->max_queues; q_idx++) {
virtqueue = &vsession->virtqueue[q_idx];
@ -340,19 +338,33 @@ spdk_vhost_session_used_signal(struct spdk_vhost_session *vsession)
}
}
static int
spdk_vhost_session_set_coalescing(struct spdk_vhost_dev *vdev,
struct spdk_vhost_session *vsession, void *ctx)
{
if (vdev == NULL || vsession == NULL) {
/* nothing to do */
return 0;
}
vsession->coalescing_delay_time_base = vdev->coalescing_delay_time_base;
vsession->coalescing_io_rate_threshold = vdev->coalescing_io_rate_threshold;
return 0;
}
int
spdk_vhost_set_coalescing(struct spdk_vhost_dev *vdev, uint32_t delay_base_us,
uint32_t iops_threshold)
{
uint64_t delay_time_base = delay_base_us * spdk_get_ticks_hz() / 1000000ULL;
uint32_t io_rate = iops_threshold * SPDK_VHOST_DEV_STATS_CHECK_INTERVAL_MS / 1000U;
uint32_t io_rate = iops_threshold * SPDK_VHOST_STATS_CHECK_INTERVAL_MS / 1000U;
if (delay_time_base >= UINT32_MAX) {
SPDK_ERRLOG("Delay time of %"PRIu32" is to big\n", delay_base_us);
return -EINVAL;
} else if (io_rate == 0) {
SPDK_ERRLOG("IOPS rate of %"PRIu32" is too low. Min is %u\n", io_rate,
1000U / SPDK_VHOST_DEV_STATS_CHECK_INTERVAL_MS);
1000U / SPDK_VHOST_STATS_CHECK_INTERVAL_MS);
return -EINVAL;
}
@ -361,6 +373,8 @@ spdk_vhost_set_coalescing(struct spdk_vhost_dev *vdev, uint32_t delay_base_us,
vdev->coalescing_delay_us = delay_base_us;
vdev->coalescing_iops_threshold = iops_threshold;
spdk_vhost_dev_foreach_session(vdev, spdk_vhost_session_set_coalescing, NULL);
return 0;
}
@ -757,9 +771,6 @@ spdk_vhost_dev_register(struct spdk_vhost_dev *vdev, const char *name, const cha
spdk_vhost_set_coalescing(vdev, SPDK_VHOST_COALESCING_DELAY_BASE_US,
SPDK_VHOST_VQ_IOPS_COALESCING_THRESHOLD);
vdev->next_stats_check_time = 0;
vdev->stats_check_interval = SPDK_VHOST_DEV_STATS_CHECK_INTERVAL_MS * spdk_get_ticks_hz() /
1000UL;
TAILQ_INSERT_TAIL(&g_spdk_vhost_devices, vdev, tailq);
@ -1130,6 +1141,7 @@ start_device(int vid)
}
}
spdk_vhost_session_set_coalescing(vdev, vsession, NULL);
vdev->lcore = spdk_vhost_allocate_reactor(vdev->cpumask);
spdk_vhost_session_mem_register(vsession);
rc = _spdk_vhost_event_send(vdev, vdev->backend->start_device, 3, "start device");
@ -1282,6 +1294,9 @@ new_connection(int vid)
vsession->vdev = vdev;
vsession->vid = vid;
vsession->next_stats_check_time = 0;
vsession->stats_check_interval = SPDK_VHOST_STATS_CHECK_INTERVAL_MS *
spdk_get_ticks_hz() / 1000UL;
vdev->session = vsession;
pthread_mutex_unlock(&g_spdk_vhost_mutex);
return 0;

18
lib/vhost/vhost_internal.h
View File

@ -70,7 +70,7 @@
/*
* Rate at which stats are checked for interrupt coalescing.
*/
#define SPDK_VHOST_DEV_STATS_CHECK_INTERVAL_MS 10
#define SPDK_VHOST_STATS_CHECK_INTERVAL_MS 10
/*
* Default threshold at which interrupts start to be coalesced.
*/
@ -147,6 +147,16 @@ struct spdk_vhost_session {
uint64_t negotiated_features;
/* Local copy of device coalescing settings. */
uint32_t coalescing_delay_time_base;
uint32_t coalescing_io_rate_threshold;
/* Next time when stats for event coalescing will be checked. */
uint64_t next_stats_check_time;
/* Interval used for event coalescing checking. */
uint64_t stats_check_interval;
struct spdk_vhost_virtqueue virtqueue[SPDK_VHOST_MAX_VQUEUES];
};
@ -174,12 +184,6 @@ struct spdk_vhost_dev {
/* Threshold when event coalescing for virtqueue will be turned on. */
uint32_t coalescing_io_rate_threshold;
/* Next time when stats for event coalescing will be checked. */
uint64_t next_stats_check_time;
/* Interval used for event coalescing checking. */
uint64_t stats_check_interval;
/* Active connection to the device */
struct spdk_vhost_session *session;