8a36da99de
Mainly focus on files that use BSD 2-Clause license, however the tool I was using misidentified many licenses so this was mostly a manual - error prone - task. The Software Package Data Exchange (SPDX) group provides a specification to make it easier for automated tools to detect and summarize well known opensource licenses. We are gradually adopting the specification, noting that the tags are considered only advisory and do not, in any way, superceed or replace the license texts.
544 lines
14 KiB
C
544 lines
14 KiB
C
/*-
|
|
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
|
*
|
|
* Copyright (c) 2001 Jake Burkholder <jake@FreeBSD.org>
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
|
|
#include <sys/cdefs.h>
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
#include "opt_sched.h"
|
|
|
|
#include <sys/param.h>
|
|
#include <sys/systm.h>
|
|
#include <sys/kdb.h>
|
|
#include <sys/kernel.h>
|
|
#include <sys/ktr.h>
|
|
#include <sys/lock.h>
|
|
#include <sys/mutex.h>
|
|
#include <sys/proc.h>
|
|
#include <sys/queue.h>
|
|
#include <sys/sched.h>
|
|
#include <sys/smp.h>
|
|
#include <sys/sysctl.h>
|
|
|
|
#include <machine/cpu.h>
|
|
|
|
/* Uncomment this to enable logging of critical_enter/exit. */
|
|
#if 0
|
|
#define KTR_CRITICAL KTR_SCHED
|
|
#else
|
|
#define KTR_CRITICAL 0
|
|
#endif
|
|
|
|
#ifdef FULL_PREEMPTION
|
|
#ifndef PREEMPTION
|
|
#error "The FULL_PREEMPTION option requires the PREEMPTION option"
|
|
#endif
|
|
#endif
|
|
|
|
CTASSERT((RQB_BPW * RQB_LEN) == RQ_NQS);
|
|
|
|
/*
|
|
* kern.sched.preemption allows user space to determine if preemption support
|
|
* is compiled in or not. It is not currently a boot or runtime flag that
|
|
* can be changed.
|
|
*/
|
|
#ifdef PREEMPTION
|
|
static int kern_sched_preemption = 1;
|
|
#else
|
|
static int kern_sched_preemption = 0;
|
|
#endif
|
|
SYSCTL_INT(_kern_sched, OID_AUTO, preemption, CTLFLAG_RD,
|
|
&kern_sched_preemption, 0, "Kernel preemption enabled");
|
|
|
|
/*
|
|
* Support for scheduler stats exported via kern.sched.stats. All stats may
|
|
* be reset with kern.sched.stats.reset = 1. Stats may be defined elsewhere
|
|
* with SCHED_STAT_DEFINE().
|
|
*/
|
|
#ifdef SCHED_STATS
|
|
SYSCTL_NODE(_kern_sched, OID_AUTO, stats, CTLFLAG_RW, 0, "switch stats");
|
|
|
|
/* Switch reasons from mi_switch(). */
|
|
DPCPU_DEFINE(long, sched_switch_stats[SWT_COUNT]);
|
|
SCHED_STAT_DEFINE_VAR(uncategorized,
|
|
&DPCPU_NAME(sched_switch_stats[SWT_NONE]), "");
|
|
SCHED_STAT_DEFINE_VAR(preempt,
|
|
&DPCPU_NAME(sched_switch_stats[SWT_PREEMPT]), "");
|
|
SCHED_STAT_DEFINE_VAR(owepreempt,
|
|
&DPCPU_NAME(sched_switch_stats[SWT_OWEPREEMPT]), "");
|
|
SCHED_STAT_DEFINE_VAR(turnstile,
|
|
&DPCPU_NAME(sched_switch_stats[SWT_TURNSTILE]), "");
|
|
SCHED_STAT_DEFINE_VAR(sleepq,
|
|
&DPCPU_NAME(sched_switch_stats[SWT_SLEEPQ]), "");
|
|
SCHED_STAT_DEFINE_VAR(sleepqtimo,
|
|
&DPCPU_NAME(sched_switch_stats[SWT_SLEEPQTIMO]), "");
|
|
SCHED_STAT_DEFINE_VAR(relinquish,
|
|
&DPCPU_NAME(sched_switch_stats[SWT_RELINQUISH]), "");
|
|
SCHED_STAT_DEFINE_VAR(needresched,
|
|
&DPCPU_NAME(sched_switch_stats[SWT_NEEDRESCHED]), "");
|
|
SCHED_STAT_DEFINE_VAR(idle,
|
|
&DPCPU_NAME(sched_switch_stats[SWT_IDLE]), "");
|
|
SCHED_STAT_DEFINE_VAR(iwait,
|
|
&DPCPU_NAME(sched_switch_stats[SWT_IWAIT]), "");
|
|
SCHED_STAT_DEFINE_VAR(suspend,
|
|
&DPCPU_NAME(sched_switch_stats[SWT_SUSPEND]), "");
|
|
SCHED_STAT_DEFINE_VAR(remotepreempt,
|
|
&DPCPU_NAME(sched_switch_stats[SWT_REMOTEPREEMPT]), "");
|
|
SCHED_STAT_DEFINE_VAR(remotewakeidle,
|
|
&DPCPU_NAME(sched_switch_stats[SWT_REMOTEWAKEIDLE]), "");
|
|
|
|
static int
|
|
sysctl_stats_reset(SYSCTL_HANDLER_ARGS)
|
|
{
|
|
struct sysctl_oid *p;
|
|
uintptr_t counter;
|
|
int error;
|
|
int val;
|
|
int i;
|
|
|
|
val = 0;
|
|
error = sysctl_handle_int(oidp, &val, 0, req);
|
|
if (error != 0 || req->newptr == NULL)
|
|
return (error);
|
|
if (val == 0)
|
|
return (0);
|
|
/*
|
|
* Traverse the list of children of _kern_sched_stats and reset each
|
|
* to 0. Skip the reset entry.
|
|
*/
|
|
SLIST_FOREACH(p, oidp->oid_parent, oid_link) {
|
|
if (p == oidp || p->oid_arg1 == NULL)
|
|
continue;
|
|
counter = (uintptr_t)p->oid_arg1;
|
|
CPU_FOREACH(i) {
|
|
*(long *)(dpcpu_off[i] + counter) = 0;
|
|
}
|
|
}
|
|
return (0);
|
|
}
|
|
|
|
SYSCTL_PROC(_kern_sched_stats, OID_AUTO, reset, CTLTYPE_INT | CTLFLAG_WR, NULL,
|
|
0, sysctl_stats_reset, "I", "Reset scheduler statistics");
|
|
#endif
|
|
|
|
/************************************************************************
|
|
* Functions that manipulate runnability from a thread perspective. *
|
|
************************************************************************/
|
|
/*
|
|
* Select the thread that will be run next.
|
|
*/
|
|
|
|
static __noinline struct thread *
|
|
choosethread_panic(struct thread *td)
|
|
{
|
|
|
|
/*
|
|
* If we are in panic, only allow system threads,
|
|
* plus the one we are running in, to be run.
|
|
*/
|
|
retry:
|
|
if (((td->td_proc->p_flag & P_SYSTEM) == 0 &&
|
|
(td->td_flags & TDF_INPANIC) == 0)) {
|
|
/* note that it is no longer on the run queue */
|
|
TD_SET_CAN_RUN(td);
|
|
td = sched_choose();
|
|
goto retry;
|
|
}
|
|
|
|
TD_SET_RUNNING(td);
|
|
return (td);
|
|
}
|
|
|
|
struct thread *
|
|
choosethread(void)
|
|
{
|
|
struct thread *td;
|
|
|
|
td = sched_choose();
|
|
|
|
if (__predict_false(panicstr != NULL))
|
|
return (choosethread_panic(td));
|
|
|
|
TD_SET_RUNNING(td);
|
|
return (td);
|
|
}
|
|
|
|
/*
|
|
* Kernel thread preemption implementation. Critical sections mark
|
|
* regions of code in which preemptions are not allowed.
|
|
*
|
|
* It might seem a good idea to inline critical_enter() but, in order
|
|
* to prevent instructions reordering by the compiler, a __compiler_membar()
|
|
* would have to be used here (the same as sched_pin()). The performance
|
|
* penalty imposed by the membar could, then, produce slower code than
|
|
* the function call itself, for most cases.
|
|
*/
|
|
void
|
|
critical_enter(void)
|
|
{
|
|
struct thread *td;
|
|
|
|
td = curthread;
|
|
td->td_critnest++;
|
|
CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td,
|
|
(long)td->td_proc->p_pid, td->td_name, td->td_critnest);
|
|
}
|
|
|
|
void
|
|
critical_exit(void)
|
|
{
|
|
struct thread *td;
|
|
int flags;
|
|
|
|
td = curthread;
|
|
KASSERT(td->td_critnest != 0,
|
|
("critical_exit: td_critnest == 0"));
|
|
|
|
if (td->td_critnest == 1) {
|
|
td->td_critnest = 0;
|
|
|
|
/*
|
|
* Interrupt handlers execute critical_exit() on
|
|
* leave, and td_owepreempt may be left set by an
|
|
* interrupt handler only when td_critnest > 0. If we
|
|
* are decrementing td_critnest from 1 to 0, read
|
|
* td_owepreempt after decrementing, to not miss the
|
|
* preempt. Disallow compiler to reorder operations.
|
|
*/
|
|
__compiler_membar();
|
|
if (td->td_owepreempt && !kdb_active) {
|
|
/*
|
|
* Microoptimization: we committed to switch,
|
|
* disable preemption in interrupt handlers
|
|
* while spinning for the thread lock.
|
|
*/
|
|
td->td_critnest = 1;
|
|
thread_lock(td);
|
|
td->td_critnest--;
|
|
flags = SW_INVOL | SW_PREEMPT;
|
|
if (TD_IS_IDLETHREAD(td))
|
|
flags |= SWT_IDLE;
|
|
else
|
|
flags |= SWT_OWEPREEMPT;
|
|
mi_switch(flags, NULL);
|
|
thread_unlock(td);
|
|
}
|
|
} else
|
|
td->td_critnest--;
|
|
|
|
CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td,
|
|
(long)td->td_proc->p_pid, td->td_name, td->td_critnest);
|
|
}
|
|
|
|
/************************************************************************
|
|
* SYSTEM RUN QUEUE manipulations and tests *
|
|
************************************************************************/
|
|
/*
|
|
* Initialize a run structure.
|
|
*/
|
|
void
|
|
runq_init(struct runq *rq)
|
|
{
|
|
int i;
|
|
|
|
bzero(rq, sizeof *rq);
|
|
for (i = 0; i < RQ_NQS; i++)
|
|
TAILQ_INIT(&rq->rq_queues[i]);
|
|
}
|
|
|
|
/*
|
|
* Clear the status bit of the queue corresponding to priority level pri,
|
|
* indicating that it is empty.
|
|
*/
|
|
static __inline void
|
|
runq_clrbit(struct runq *rq, int pri)
|
|
{
|
|
struct rqbits *rqb;
|
|
|
|
rqb = &rq->rq_status;
|
|
CTR4(KTR_RUNQ, "runq_clrbit: bits=%#x %#x bit=%#x word=%d",
|
|
rqb->rqb_bits[RQB_WORD(pri)],
|
|
rqb->rqb_bits[RQB_WORD(pri)] & ~RQB_BIT(pri),
|
|
RQB_BIT(pri), RQB_WORD(pri));
|
|
rqb->rqb_bits[RQB_WORD(pri)] &= ~RQB_BIT(pri);
|
|
}
|
|
|
|
/*
|
|
* Find the index of the first non-empty run queue. This is done by
|
|
* scanning the status bits, a set bit indicates a non-empty queue.
|
|
*/
|
|
static __inline int
|
|
runq_findbit(struct runq *rq)
|
|
{
|
|
struct rqbits *rqb;
|
|
int pri;
|
|
int i;
|
|
|
|
rqb = &rq->rq_status;
|
|
for (i = 0; i < RQB_LEN; i++)
|
|
if (rqb->rqb_bits[i]) {
|
|
pri = RQB_FFS(rqb->rqb_bits[i]) + (i << RQB_L2BPW);
|
|
CTR3(KTR_RUNQ, "runq_findbit: bits=%#x i=%d pri=%d",
|
|
rqb->rqb_bits[i], i, pri);
|
|
return (pri);
|
|
}
|
|
|
|
return (-1);
|
|
}
|
|
|
|
static __inline int
|
|
runq_findbit_from(struct runq *rq, u_char pri)
|
|
{
|
|
struct rqbits *rqb;
|
|
rqb_word_t mask;
|
|
int i;
|
|
|
|
/*
|
|
* Set the mask for the first word so we ignore priorities before 'pri'.
|
|
*/
|
|
mask = (rqb_word_t)-1 << (pri & (RQB_BPW - 1));
|
|
rqb = &rq->rq_status;
|
|
again:
|
|
for (i = RQB_WORD(pri); i < RQB_LEN; mask = -1, i++) {
|
|
mask = rqb->rqb_bits[i] & mask;
|
|
if (mask == 0)
|
|
continue;
|
|
pri = RQB_FFS(mask) + (i << RQB_L2BPW);
|
|
CTR3(KTR_RUNQ, "runq_findbit_from: bits=%#x i=%d pri=%d",
|
|
mask, i, pri);
|
|
return (pri);
|
|
}
|
|
if (pri == 0)
|
|
return (-1);
|
|
/*
|
|
* Wrap back around to the beginning of the list just once so we
|
|
* scan the whole thing.
|
|
*/
|
|
pri = 0;
|
|
goto again;
|
|
}
|
|
|
|
/*
|
|
* Set the status bit of the queue corresponding to priority level pri,
|
|
* indicating that it is non-empty.
|
|
*/
|
|
static __inline void
|
|
runq_setbit(struct runq *rq, int pri)
|
|
{
|
|
struct rqbits *rqb;
|
|
|
|
rqb = &rq->rq_status;
|
|
CTR4(KTR_RUNQ, "runq_setbit: bits=%#x %#x bit=%#x word=%d",
|
|
rqb->rqb_bits[RQB_WORD(pri)],
|
|
rqb->rqb_bits[RQB_WORD(pri)] | RQB_BIT(pri),
|
|
RQB_BIT(pri), RQB_WORD(pri));
|
|
rqb->rqb_bits[RQB_WORD(pri)] |= RQB_BIT(pri);
|
|
}
|
|
|
|
/*
|
|
* Add the thread to the queue specified by its priority, and set the
|
|
* corresponding status bit.
|
|
*/
|
|
void
|
|
runq_add(struct runq *rq, struct thread *td, int flags)
|
|
{
|
|
struct rqhead *rqh;
|
|
int pri;
|
|
|
|
pri = td->td_priority / RQ_PPQ;
|
|
td->td_rqindex = pri;
|
|
runq_setbit(rq, pri);
|
|
rqh = &rq->rq_queues[pri];
|
|
CTR4(KTR_RUNQ, "runq_add: td=%p pri=%d %d rqh=%p",
|
|
td, td->td_priority, pri, rqh);
|
|
if (flags & SRQ_PREEMPTED) {
|
|
TAILQ_INSERT_HEAD(rqh, td, td_runq);
|
|
} else {
|
|
TAILQ_INSERT_TAIL(rqh, td, td_runq);
|
|
}
|
|
}
|
|
|
|
void
|
|
runq_add_pri(struct runq *rq, struct thread *td, u_char pri, int flags)
|
|
{
|
|
struct rqhead *rqh;
|
|
|
|
KASSERT(pri < RQ_NQS, ("runq_add_pri: %d out of range", pri));
|
|
td->td_rqindex = pri;
|
|
runq_setbit(rq, pri);
|
|
rqh = &rq->rq_queues[pri];
|
|
CTR4(KTR_RUNQ, "runq_add_pri: td=%p pri=%d idx=%d rqh=%p",
|
|
td, td->td_priority, pri, rqh);
|
|
if (flags & SRQ_PREEMPTED) {
|
|
TAILQ_INSERT_HEAD(rqh, td, td_runq);
|
|
} else {
|
|
TAILQ_INSERT_TAIL(rqh, td, td_runq);
|
|
}
|
|
}
|
|
/*
|
|
* Return true if there are runnable processes of any priority on the run
|
|
* queue, false otherwise. Has no side effects, does not modify the run
|
|
* queue structure.
|
|
*/
|
|
int
|
|
runq_check(struct runq *rq)
|
|
{
|
|
struct rqbits *rqb;
|
|
int i;
|
|
|
|
rqb = &rq->rq_status;
|
|
for (i = 0; i < RQB_LEN; i++)
|
|
if (rqb->rqb_bits[i]) {
|
|
CTR2(KTR_RUNQ, "runq_check: bits=%#x i=%d",
|
|
rqb->rqb_bits[i], i);
|
|
return (1);
|
|
}
|
|
CTR0(KTR_RUNQ, "runq_check: empty");
|
|
|
|
return (0);
|
|
}
|
|
|
|
/*
|
|
* Find the highest priority process on the run queue.
|
|
*/
|
|
struct thread *
|
|
runq_choose_fuzz(struct runq *rq, int fuzz)
|
|
{
|
|
struct rqhead *rqh;
|
|
struct thread *td;
|
|
int pri;
|
|
|
|
while ((pri = runq_findbit(rq)) != -1) {
|
|
rqh = &rq->rq_queues[pri];
|
|
/* fuzz == 1 is normal.. 0 or less are ignored */
|
|
if (fuzz > 1) {
|
|
/*
|
|
* In the first couple of entries, check if
|
|
* there is one for our CPU as a preference.
|
|
*/
|
|
int count = fuzz;
|
|
int cpu = PCPU_GET(cpuid);
|
|
struct thread *td2;
|
|
td2 = td = TAILQ_FIRST(rqh);
|
|
|
|
while (count-- && td2) {
|
|
if (td2->td_lastcpu == cpu) {
|
|
td = td2;
|
|
break;
|
|
}
|
|
td2 = TAILQ_NEXT(td2, td_runq);
|
|
}
|
|
} else
|
|
td = TAILQ_FIRST(rqh);
|
|
KASSERT(td != NULL, ("runq_choose_fuzz: no proc on busy queue"));
|
|
CTR3(KTR_RUNQ,
|
|
"runq_choose_fuzz: pri=%d thread=%p rqh=%p", pri, td, rqh);
|
|
return (td);
|
|
}
|
|
CTR1(KTR_RUNQ, "runq_choose_fuzz: idleproc pri=%d", pri);
|
|
|
|
return (NULL);
|
|
}
|
|
|
|
/*
|
|
* Find the highest priority process on the run queue.
|
|
*/
|
|
struct thread *
|
|
runq_choose(struct runq *rq)
|
|
{
|
|
struct rqhead *rqh;
|
|
struct thread *td;
|
|
int pri;
|
|
|
|
while ((pri = runq_findbit(rq)) != -1) {
|
|
rqh = &rq->rq_queues[pri];
|
|
td = TAILQ_FIRST(rqh);
|
|
KASSERT(td != NULL, ("runq_choose: no thread on busy queue"));
|
|
CTR3(KTR_RUNQ,
|
|
"runq_choose: pri=%d thread=%p rqh=%p", pri, td, rqh);
|
|
return (td);
|
|
}
|
|
CTR1(KTR_RUNQ, "runq_choose: idlethread pri=%d", pri);
|
|
|
|
return (NULL);
|
|
}
|
|
|
|
struct thread *
|
|
runq_choose_from(struct runq *rq, u_char idx)
|
|
{
|
|
struct rqhead *rqh;
|
|
struct thread *td;
|
|
int pri;
|
|
|
|
if ((pri = runq_findbit_from(rq, idx)) != -1) {
|
|
rqh = &rq->rq_queues[pri];
|
|
td = TAILQ_FIRST(rqh);
|
|
KASSERT(td != NULL, ("runq_choose: no thread on busy queue"));
|
|
CTR4(KTR_RUNQ,
|
|
"runq_choose_from: pri=%d thread=%p idx=%d rqh=%p",
|
|
pri, td, td->td_rqindex, rqh);
|
|
return (td);
|
|
}
|
|
CTR1(KTR_RUNQ, "runq_choose_from: idlethread pri=%d", pri);
|
|
|
|
return (NULL);
|
|
}
|
|
/*
|
|
* Remove the thread from the queue specified by its priority, and clear the
|
|
* corresponding status bit if the queue becomes empty.
|
|
* Caller must set state afterwards.
|
|
*/
|
|
void
|
|
runq_remove(struct runq *rq, struct thread *td)
|
|
{
|
|
|
|
runq_remove_idx(rq, td, NULL);
|
|
}
|
|
|
|
void
|
|
runq_remove_idx(struct runq *rq, struct thread *td, u_char *idx)
|
|
{
|
|
struct rqhead *rqh;
|
|
u_char pri;
|
|
|
|
KASSERT(td->td_flags & TDF_INMEM,
|
|
("runq_remove_idx: thread swapped out"));
|
|
pri = td->td_rqindex;
|
|
KASSERT(pri < RQ_NQS, ("runq_remove_idx: Invalid index %d\n", pri));
|
|
rqh = &rq->rq_queues[pri];
|
|
CTR4(KTR_RUNQ, "runq_remove_idx: td=%p, pri=%d %d rqh=%p",
|
|
td, td->td_priority, pri, rqh);
|
|
TAILQ_REMOVE(rqh, td, td_runq);
|
|
if (TAILQ_EMPTY(rqh)) {
|
|
CTR0(KTR_RUNQ, "runq_remove_idx: empty");
|
|
runq_clrbit(rq, pri);
|
|
if (idx != NULL && *idx == pri)
|
|
*idx = (pri + 1) % RQ_NQS;
|
|
}
|
|
}
|