From ed8d4e870343bc171f4834393b60df199d19aaef Mon Sep 17 00:00:00 2001 From: jeff Date: Sun, 25 Jan 2004 08:00:04 +0000 Subject: [PATCH] - Implement cpu pinning and binding. This is acomplished by keeping a per- cpu run queue that is only used for pinned or bound threads. Submitted by: Chris Bradfield --- sys/kern/sched_4bsd.c | 140 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 128 insertions(+), 12 deletions(-) diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c index 9fa64d3cedcc..24b661d2ab18 100644 --- a/sys/kern/sched_4bsd.c +++ b/sys/kern/sched_4bsd.c @@ -53,6 +53,8 @@ __FBSDID("$FreeBSD$"); #include #include +#define KTR_4BSD 0x0 + /* * INVERSE_ESTCPU_WEIGHT is only suitable for statclock() frequencies in * the range 100-256 Hz (approximately). @@ -68,9 +70,29 @@ __FBSDID("$FreeBSD$"); #define NICE_WEIGHT 1 /* Priorities per nice level. */ struct ke_sched { - int ske_cpticks; /* (j) Ticks of cpu time. */ + int ske_cpticks; /* (j) Ticks of cpu time. */ + struct runq *ske_runq; /* runq the kse is currently on */ }; +#define ke_runq ke_sched->ske_runq +#define KEF_BOUND KEF_SCHED1 +#define SKE_RUNQ_PCPU(ke) \ + ((ke)->ke_runq != 0 && (ke)->ke_runq != &runq) + +/* + * KSE_CAN_MIGRATE macro returns true if the kse can migrate between + * cpus. Currently ithread cpu binding is disabled on x86 due to a + * bug in the Xeon round-robin interrupt delivery that delivers all + * interrupts to cpu 0. + */ +#ifdef __i386__ +#define KSE_CAN_MIGRATE(ke) \ + ((ke)->ke_thread->td_pinned == 0 && ((ke)->ke_flags & KEF_BOUND) == 0) +#else +#define KSE_CAN_MIGRATE(ke) \ + PRI_BASE((ke)->ke_ksegrp->kg_pri_class) != PRI_ITHD && \ + ((ke)->ke_thread->td_pinned == 0 &&((ke)->ke_flags & KEF_BOUND) == 0) +#endif static struct ke_sched ke_sched; struct ke_sched *kse0_sched = &ke_sched; @@ -83,21 +105,47 @@ static int sched_quantum; /* Roundrobin scheduling quantum in ticks. */ static struct callout roundrobin_callout; +static void setup_runqs(void); static void roundrobin(void *arg); static void schedcpu(void); -static void schedcpu_thread(void *dummy); +static void schedcpu_thread(void); static void sched_setup(void *dummy); static void maybe_resched(struct thread *td); static void updatepri(struct ksegrp *kg); static void resetpriority(struct ksegrp *kg); -SYSINIT(sched_setup, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, sched_setup, NULL) +static struct kproc_desc sched_kp = { + "schedcpu", + schedcpu_thread, + NULL +}; +SYSINIT(schedcpu, SI_SUB_RUN_SCHEDULER, SI_ORDER_FIRST, kproc_start, &sched_kp) +SYSINIT(sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL) /* * Global run queue. */ static struct runq runq; -SYSINIT(runq, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, runq_init, &runq) + +#ifdef SMP +/* + * Per-CPU run queues + */ +static struct runq runq_pcpu[MAXCPU]; +#endif + +static void +setup_runqs(void) +{ +#ifdef SMP + int i; + + for (i = 0; i < MAXCPU; ++i) + runq_init(&runq_pcpu[i]); +#endif + + runq_init(&runq); +} static int sysctl_kern_quantum(SYSCTL_HANDLER_ARGS) @@ -355,7 +403,7 @@ schedcpu(void) * Main loop for a kthread that executes schedcpu once a second. */ static void -schedcpu_thread(void *dummy) +schedcpu_thread(void) { int nowake; @@ -416,6 +464,7 @@ resetpriority(struct ksegrp *kg) static void sched_setup(void *dummy) { + setup_runqs(); if (sched_quantum == 0) sched_quantum = SCHED_QUANTUM; @@ -425,16 +474,17 @@ sched_setup(void *dummy) /* Kick off timeout driven events by calling first time. */ roundrobin(NULL); - - /* Kick off schedcpu kernel process. */ - kthread_create(schedcpu_thread, NULL, NULL, 0, 0, "schedcpu"); } /* External interfaces start here */ int sched_runnable(void) { - return runq_check(&runq); +#ifdef SMP + return runq_check(&runq) + runq_check(&runq_pcpu[PCPU_GET(cpuid)]); +#else + return runq_check(&runq); +#endif } int @@ -654,7 +704,20 @@ sched_add(struct thread *td) ke->ke_ksegrp->kg_runq_kses++; ke->ke_state = KES_ONRUNQ; - runq_add(&runq, ke); +#ifdef SMP + if (KSE_CAN_MIGRATE(ke)) { + CTR1(KTR_4BSD, "adding kse:%p to gbl runq", ke); + ke->ke_runq = &runq; + } else { + CTR1(KTR_4BSD, "adding kse:%p to pcpu runq", ke); + if (!SKE_RUNQ_PCPU(ke)) + ke->ke_runq = &runq_pcpu[PCPU_GET(cpuid)]; + } +#else + ke->ke_runq = &runq; +#endif + + runq_add(ke->ke_runq, ke); } void @@ -668,7 +731,8 @@ sched_rem(struct thread *td) KASSERT((ke->ke_state == KES_ONRUNQ), ("KSE not on run queue")); mtx_assert(&sched_lock, MA_OWNED); - runq_remove(&runq, ke); + runq_remove(ke->ke_sched->ske_runq, ke); + ke->ke_state = KES_THREAD; ke->ke_ksegrp->kg_runq_kses--; } @@ -677,11 +741,33 @@ struct kse * sched_choose(void) { struct kse *ke; + struct runq *rq; +#ifdef SMP + struct kse *kecpu; + + rq = &runq; ke = runq_choose(&runq); + kecpu = runq_choose(&runq_pcpu[PCPU_GET(cpuid)]); + + if (ke == NULL || + (kecpu != NULL && + kecpu->ke_thread->td_priority < ke->ke_thread->td_priority)) { + CTR2(KTR_4BSD, "choosing kse %p from pcpu runq %d", kecpu, + PCPU_GET(cpuid)); + ke = kecpu; + rq = &runq_pcpu[PCPU_GET(cpuid)]; + } else { + CTR1(KTR_4BSD, "choosing kse %p from main runq", ke); + } + +#else + rq = &runq; + ke = runq_choose(&runq); +#endif if (ke != NULL) { - runq_remove(&runq, ke); + runq_remove(rq, ke); ke->ke_state = KES_THREAD; KASSERT((ke->ke_thread != NULL), @@ -715,6 +801,36 @@ sched_userret(struct thread *td) } } +void +sched_bind(struct thread *td, int cpu) +{ + struct kse *ke; + + mtx_assert(&sched_lock, MA_OWNED); + KASSERT(TD_IS_RUNNING(td), + ("sched_bind: cannot bind non-running thread")); + + ke = td->td_kse; + + ke->ke_flags |= KEF_BOUND; +#ifdef SMP + ke->ke_runq = &runq_pcpu[cpu]; + if (PCPU_GET(cpuid) == cpu) + return; + + ke->ke_state = KES_THREAD; + + mi_switch(SW_VOL); +#endif +} + +void +sched_unbind(struct thread* td) +{ + mtx_assert(&sched_lock, MA_OWNED); + td->td_kse->ke_flags &= ~KEF_BOUND; +} + int sched_sizeof_kse(void) {