From cf494305bf6494ff2c167259f4d487f08b2f66de Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Wed, 22 Sep 2021 09:29:45 -0700 Subject: [PATCH] scheduler/dynamic: don't put dst core over limit when moving thread We have _is_core_over_limit() which determines if a core is currently over its busy:total tsc ratio. We use this to determine if we need to move threads off of a core that is too busy. But when we pick a core to move a thread *to* we were allowing the dst core to fill to 100%, rather than the SCHEDULER_CORE_LIMIT. This patch fixes that, which has the nice effect of keeping thread-to-core assignments much more stable when running I/O workloads. Signed-off-by: Jim Harris Change-Id: Id98b08803939d2a25104082e6436bb8d4727d7c2 Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/9578 Community-CI: Broadcom CI Community-CI: Mellanox Build Bot Reviewed-by: Ben Walker Reviewed-by: Tomasz Zawadzki Tested-by: SPDK CI Jenkins --- module/scheduler/dynamic/scheduler_dynamic.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/module/scheduler/dynamic/scheduler_dynamic.c b/module/scheduler/dynamic/scheduler_dynamic.c index 7d5ae4271c..1d5bf424f5 100644 --- a/module/scheduler/dynamic/scheduler_dynamic.c +++ b/module/scheduler/dynamic/scheduler_dynamic.c @@ -169,6 +169,7 @@ static bool _can_core_fit_thread(struct spdk_scheduler_thread_info *thread_info, uint32_t dst_core) { struct core_stats *dst = &g_cores[dst_core]; + uint64_t new_busy_tsc, new_idle_tsc; /* Thread can always fit on the core it's currently on. */ if (thread_info->lcore == dst_core) { @@ -186,10 +187,17 @@ _can_core_fit_thread(struct spdk_scheduler_thread_info *thread_info, uint32_t ds return true; } - if (thread_info->current_stats.busy_tsc <= dst->idle) { - return true; + /* Core doesn't have enough idle_tsc to take this thread. */ + if (dst->idle < thread_info->current_stats.busy_tsc) { + return false; } - return false; + + new_busy_tsc = dst->busy + thread_info->current_stats.busy_tsc; + new_idle_tsc = dst->idle - thread_info->current_stats.busy_tsc; + + /* Core cannot fit this thread if it would put it over the + * SCHEDULER_CORE_LIMIT. */ + return _busy_pct(new_busy_tsc, new_idle_tsc) < SCHEDULER_CORE_LIMIT; } static uint32_t