From 6df35af4d85c6311d8e762521580e7176b69394e Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Fri, 25 Jun 2021 13:52:58 -0400 Subject: [PATCH] Allow sleepq_signal() to drop the lock. Introduce SLEEPQ_DROP sleepq_signal() flag, allowing one to drop the sleep queue chain lock before returning. Reduced lock scope allows significantly reduce lock contention inside taskqueue_enqueue() for ZFS worker threads doing ~350K disk reads/s on 40-thread system. MFC after: 2 weeks Sponsored by: iXsystems, Inc. --- sys/kern/kern_synch.c | 8 +++----- sys/kern/subr_sleepqueue.c | 8 ++++++-- sys/sys/sleepqueue.h | 1 + 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index dcca67326264..793c5309a30b 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -366,8 +366,7 @@ wakeup_one(const void *ident) int wakeup_swapper; sleepq_lock(ident); - wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP, 0, 0); - sleepq_release(ident); + wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP | SLEEPQ_DROP, 0, 0); if (wakeup_swapper) kick_proc0(); } @@ -378,9 +377,8 @@ wakeup_any(const void *ident) int wakeup_swapper; sleepq_lock(ident); - wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP | SLEEPQ_UNFAIR, - 0, 0); - sleepq_release(ident); + wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP | SLEEPQ_UNFAIR | + SLEEPQ_DROP, 0, 0); if (wakeup_swapper) kick_proc0(); } diff --git a/sys/kern/subr_sleepqueue.c b/sys/kern/subr_sleepqueue.c index 0718f01fa48a..b146a978a60c 100644 --- a/sys/kern/subr_sleepqueue.c +++ b/sys/kern/subr_sleepqueue.c @@ -927,8 +927,11 @@ sleepq_signal(const void *wchan, int flags, int pri, int queue) KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__)); MPASS((queue >= 0) && (queue < NR_SLEEPQS)); sq = sleepq_lookup(wchan); - if (sq == NULL) + if (sq == NULL) { + if (flags & SLEEPQ_DROP) + sleepq_release(wchan); return (0); + } KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE), ("%s: mismatch between sleep/wakeup and cv_*", __func__)); @@ -961,7 +964,8 @@ sleepq_signal(const void *wchan, int flags, int pri, int queue) } } MPASS(besttd != NULL); - wakeup_swapper = sleepq_resume_thread(sq, besttd, pri, SRQ_HOLD); + wakeup_swapper = sleepq_resume_thread(sq, besttd, pri, + (flags & SLEEPQ_DROP) ? 0 : SRQ_HOLD); return (wakeup_swapper); } diff --git a/sys/sys/sleepqueue.h b/sys/sys/sleepqueue.h index 18c7568777b6..6715894ed1f3 100644 --- a/sys/sys/sleepqueue.h +++ b/sys/sys/sleepqueue.h @@ -85,6 +85,7 @@ struct thread; #define SLEEPQ_LK 0x04 /* Used by a lockmgr. */ #define SLEEPQ_INTERRUPTIBLE 0x100 /* Sleep is interruptible. */ #define SLEEPQ_UNFAIR 0x200 /* Unfair wakeup order. */ +#define SLEEPQ_DROP 0x400 /* Return without lock held. */ void init_sleepqueues(void); int sleepq_abort(struct thread *td, int intrval);