From ad1e7d285ab101b694e6d7a0cdbf7ec10c8dbdce Mon Sep 17 00:00:00 2001 From: Julian Elischer Date: Wed, 6 Dec 2006 06:34:57 +0000 Subject: [PATCH] Threading cleanup.. part 2 of several. Make part of John Birrell's KSE patch permanent.. Specifically, remove: Any reference of the ksegrp structure. This feature was never fully utilised and made things overly complicated. All code in the scheduler that tried to make threaded programs fair to unthreaded programs. Libpthread processes will already do this to some extent and libthr processes already disable it. Also: Since this makes such a big change to the scheduler(s), take the opportunity to rename some structures and elements that had to be moved anyhow. This makes the code a lot more readable. The ULE scheduler compiles again but I have no idea if it works. The 4bsd scheduler still reqires a little cleaning and some functions that now do ALMOST nothing will go away, but I thought I'd do that as a separate commit. Tested by David Xu, and Dan Eischen using libthr and libpthread. --- sys/amd64/amd64/machdep.c | 4 - sys/arm/at91/kb920x_machdep.c | 4 - sys/arm/sa11x0/assabet_machdep.c | 4 - sys/arm/xscale/i80321/ep80219_machdep.c | 4 - sys/arm/xscale/i80321/iq31244_machdep.c | 4 - sys/arm/xscale/ixp425/avila_machdep.c | 4 - sys/ddb/db_ps.c | 5 - sys/fs/procfs/procfs_status.c | 13 +- sys/i386/i386/machdep.c | 6 +- sys/ia64/ia64/machdep.c | 8 +- sys/kern/init_main.c | 32 - sys/kern/kern_clock.c | 20 +- sys/kern/kern_fork.c | 18 - sys/kern/kern_idle.c | 4 - sys/kern/kern_intr.c | 6 +- sys/kern/kern_kse.c | 331 +++---- sys/kern/kern_poll.c | 4 - sys/kern/kern_proc.c | 39 +- sys/kern/kern_resource.c | 89 -- sys/kern/kern_subr.c | 4 - sys/kern/kern_switch.c | 637 ++----------- sys/kern/kern_thr.c | 36 +- sys/kern/kern_thread.c | 254 +---- sys/kern/kern_umtx.c | 6 - sys/kern/ksched.c | 45 - sys/kern/sched_4bsd.c | 762 +++------------ sys/kern/sched_ule.c | 1125 +++++++++++------------ sys/kern/subr_trap.c | 12 +- sys/kern/tty.c | 27 +- sys/pc98/pc98/machdep.c | 4 - sys/powerpc/aim/machdep.c | 4 - sys/powerpc/powerpc/machdep.c | 4 - sys/sparc64/sparc64/machdep.c | 4 - sys/sun4v/sun4v/machdep.c | 4 - sys/sys/proc.h | 234 +---- sys/sys/rtprio.h | 6 - sys/sys/runq.h | 10 +- sys/sys/sched.h | 27 +- sys/vm/vm_glue.c | 42 +- sys/vm/vm_zeroidle.c | 4 - 40 files changed, 942 insertions(+), 2908 deletions(-) diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index f0a6239103b1..348bfef78dc3 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -1121,11 +1121,7 @@ hammer_time(u_int64_t modulep, u_int64_t physfree) * This may be done better later if it gets more high level * components in it. If so just link td->td_proc here. */ -#ifdef KSE - proc_linkup(&proc0, &ksegrp0, &thread0); -#else proc_linkup(&proc0, &thread0); -#endif preload_metadata = (caddr_t)(uintptr_t)(modulep + KERNBASE); preload_bootstrap_relocate(KERNBASE); diff --git a/sys/arm/at91/kb920x_machdep.c b/sys/arm/at91/kb920x_machdep.c index 9bff88845a39..7cb5af508730 100644 --- a/sys/arm/at91/kb920x_machdep.c +++ b/sys/arm/at91/kb920x_machdep.c @@ -455,11 +455,7 @@ initarm(void *arg, void *arg2) undefined_handler_address = (u_int)undefinedinstruction_bounce; undefined_init(); -#ifdef KSE - proc_linkup(&proc0, &ksegrp0, &thread0); -#else proc_linkup(&proc0, &thread0); -#endif thread0.td_kstack = kernelstack.pv_va; thread0.td_pcb = (struct pcb *) (thread0.td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1; diff --git a/sys/arm/sa11x0/assabet_machdep.c b/sys/arm/sa11x0/assabet_machdep.c index b8f2879c557c..482f3a54069a 100644 --- a/sys/arm/sa11x0/assabet_machdep.c +++ b/sys/arm/sa11x0/assabet_machdep.c @@ -422,11 +422,7 @@ initarm(void *arg, void *arg2) /* Set stack for exception handlers */ -#ifdef KSE - proc_linkup(&proc0, &ksegrp0, &thread0); -#else proc_linkup(&proc0, &thread0); -#endif thread0.td_kstack = kernelstack.pv_va; thread0.td_pcb = (struct pcb *) (thread0.td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1; diff --git a/sys/arm/xscale/i80321/ep80219_machdep.c b/sys/arm/xscale/i80321/ep80219_machdep.c index 75e9e1907694..9b787a666519 100644 --- a/sys/arm/xscale/i80321/ep80219_machdep.c +++ b/sys/arm/xscale/i80321/ep80219_machdep.c @@ -429,11 +429,7 @@ initarm(void *arg, void *arg2) undefined_handler_address = (u_int)undefinedinstruction_bounce; undefined_init(); -#ifdef KSE - proc_linkup(&proc0, &ksegrp0, &thread0); -#else proc_linkup(&proc0, &thread0); -#endif thread0.td_kstack = kernelstack.pv_va; thread0.td_pcb = (struct pcb *) (thread0.td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1; diff --git a/sys/arm/xscale/i80321/iq31244_machdep.c b/sys/arm/xscale/i80321/iq31244_machdep.c index 2a52c043f9c7..1af00631ece4 100644 --- a/sys/arm/xscale/i80321/iq31244_machdep.c +++ b/sys/arm/xscale/i80321/iq31244_machdep.c @@ -427,11 +427,7 @@ initarm(void *arg, void *arg2) undefined_handler_address = (u_int)undefinedinstruction_bounce; undefined_init(); -#ifdef KSE - proc_linkup(&proc0, &ksegrp0, &thread0); -#else proc_linkup(&proc0, &thread0); -#endif thread0.td_kstack = kernelstack.pv_va; thread0.td_pcb = (struct pcb *) (thread0.td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1; diff --git a/sys/arm/xscale/ixp425/avila_machdep.c b/sys/arm/xscale/ixp425/avila_machdep.c index b6efecf51576..29c3d684c07f 100644 --- a/sys/arm/xscale/ixp425/avila_machdep.c +++ b/sys/arm/xscale/ixp425/avila_machdep.c @@ -493,11 +493,7 @@ initarm(void *arg, void *arg2) undefined_handler_address = (u_int)undefinedinstruction_bounce; undefined_init(); -#ifdef KSE - proc_linkup(&proc0, &ksegrp0, &thread0); -#else proc_linkup(&proc0, &thread0); -#endif thread0.td_kstack = kernelstack.pv_va; thread0.td_pcb = (struct pcb *) (thread0.td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1; diff --git a/sys/ddb/db_ps.c b/sys/ddb/db_ps.c index 853675a911ed..61e36b94fc1b 100644 --- a/sys/ddb/db_ps.c +++ b/sys/ddb/db_ps.c @@ -292,12 +292,7 @@ DB_SHOW_COMMAND(thread, db_show_thread) td = kdb_thread; db_printf("Thread %d at %p:\n", td->td_tid, td); -#ifdef KSE - db_printf(" proc (pid %d): %p ", td->td_proc->p_pid, td->td_proc); - db_printf(" ksegrp: %p\n", td->td_ksegrp); -#else db_printf(" proc (pid %d): %p\n", td->td_proc->p_pid, td->td_proc); -#endif if (td->td_name[0] != '\0') db_printf(" name: %s\n", td->td_name); db_printf(" flags: %#x ", td->td_flags); diff --git a/sys/fs/procfs/procfs_status.c b/sys/fs/procfs/procfs_status.c index 60aa270cb6f3..63827758b113 100644 --- a/sys/fs/procfs/procfs_status.c +++ b/sys/fs/procfs/procfs_status.c @@ -116,7 +116,9 @@ procfs_doprocstatus(PFS_FILL_ARGS) #ifdef KSE if (p->p_flag & P_SA) wmesg = "-kse- "; - else { + else +#endif + { tdfirst = FIRST_THREAD_IN_PROC(p); if (tdfirst->td_wchan != NULL) { KASSERT(tdfirst->td_wmesg != NULL, @@ -125,15 +127,6 @@ procfs_doprocstatus(PFS_FILL_ARGS) } else wmesg = "nochan"; } -#else - tdfirst = FIRST_THREAD_IN_PROC(p); - if (tdfirst->td_wchan != NULL) { - KASSERT(tdfirst->td_wmesg != NULL, - ("wchan %p has no wmesg", tdfirst->td_wchan)); - wmesg = tdfirst->td_wmesg; - } else - wmesg = "nochan"; -#endif mtx_unlock_spin(&sched_lock); if (p->p_sflag & PS_INMEM) { diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c index c416c38550ed..2dc54ecc5f50 100644 --- a/sys/i386/i386/machdep.c +++ b/sys/i386/i386/machdep.c @@ -2058,11 +2058,7 @@ init386(first) * This may be done better later if it gets more high level * components in it. If so just link td->td_proc here. */ -#ifdef KSE - proc_linkup(&proc0, &ksegrp0, &thread0); -#else proc_linkup(&proc0, &thread0); -#endif metadata_missing = 0; if (bootinfo.bi_modulep) { @@ -2297,7 +2293,7 @@ init386(first) _udatasel = GSEL(GUDATA_SEL, SEL_UPL); /* setup proc 0's pcb */ - thread0.td_pcb->pcb_flags = 0; /* XXXKSE */ + thread0.td_pcb->pcb_flags = 0; #ifdef PAE thread0.td_pcb->pcb_cr3 = (int)IdlePDPT; #else diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c index a57fc4f1b48b..7c2e1f5bd7b1 100644 --- a/sys/ia64/ia64/machdep.c +++ b/sys/ia64/ia64/machdep.c @@ -776,12 +776,10 @@ ia64_init(void) msgbufp = (struct msgbuf *)pmap_steal_memory(MSGBUF_SIZE); msgbufinit(msgbufp, MSGBUF_SIZE); -#ifdef KSE - proc_linkup(&proc0, &ksegrp0, &thread0); -#else proc_linkup(&proc0, &thread0); -#endif - + /* + * Init mapping for kernel stack for proc 0 + */ proc0kstack = (vm_offset_t)kstack; thread0.td_kstack = proc0kstack; thread0.td_kstack_pages = KSTACK_PAGES; diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c index d002a99af958..a911e111e80e 100644 --- a/sys/kern/init_main.c +++ b/sys/kern/init_main.c @@ -95,9 +95,6 @@ static struct session session0; static struct pgrp pgrp0; struct proc proc0; struct thread thread0 __aligned(8); -#ifdef KSE -struct ksegrp ksegrp0; -#endif struct vmspace vmspace0; struct proc *initproc; @@ -366,34 +363,16 @@ proc0_init(void *dummy __unused) struct proc *p; unsigned i; struct thread *td; -#ifdef KSE - struct ksegrp *kg; -#endif GIANT_REQUIRED; p = &proc0; td = &thread0; -#ifdef KSE - kg = &ksegrp0; -#endif /* * Initialize magic number. */ p->p_magic = P_MAGIC; -#ifdef KSE - /* - * Initialize thread, process and ksegrp structures. - */ - procinit(); /* set up proc zone */ - threadinit(); /* set up thead, upcall and KSEGRP zones */ - - /* - * Initialise scheduler resources. - * Add scheduler specific parts to proc, ksegrp, thread as needed. - */ -#else /* * Initialize thread and process structures. */ @@ -404,7 +383,6 @@ proc0_init(void *dummy __unused) * Initialise scheduler resources. * Add scheduler specific parts to proc, thread as needed. */ -#endif schedinit(); /* scheduler gets its house in order */ /* * Initialize sleep queue hash table @@ -440,15 +418,9 @@ proc0_init(void *dummy __unused) STAILQ_INIT(&p->p_ktr); p->p_nice = NZERO; td->td_state = TDS_RUNNING; -#ifdef KSE - kg->kg_pri_class = PRI_TIMESHARE; - kg->kg_user_pri = PUSER; - kg->kg_base_user_pri = PUSER; -#else td->td_pri_class = PRI_TIMESHARE; td->td_user_pri = PUSER; td->td_base_user_pri = PUSER; -#endif td->td_priority = PVM; td->td_base_pri = PUSER; td->td_oncpu = 0; @@ -758,11 +730,7 @@ kick_init(const void *udata __unused) td = FIRST_THREAD_IN_PROC(initproc); mtx_lock_spin(&sched_lock); TD_SET_CAN_RUN(td); -#ifdef KSE - setrunqueue(td, SRQ_BORING); /* XXXKSE */ -#else setrunqueue(td, SRQ_BORING); -#endif mtx_unlock_spin(&sched_lock); } SYSINIT(kickinit, SI_SUB_KTHREAD_INIT, SI_ORDER_FIRST, kick_init, NULL) diff --git a/sys/kern/kern_clock.c b/sys/kern/kern_clock.c index cd98bdd26ee9..a3732a81e595 100644 --- a/sys/kern/kern_clock.c +++ b/sys/kern/kern_clock.c @@ -203,23 +203,12 @@ hardclock_cpu(int usermode) mtx_lock_spin_flags(&sched_lock, MTX_QUIET); sched_tick(); #ifdef KSE +#if 0 /* for now do nothing */ if (p->p_flag & P_SA) { - /* XXXKSE What to do? */ - } else { - pstats = p->p_stats; - if (usermode && - timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) && - itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) { - p->p_sflag |= PS_ALRMPEND; - td->td_flags |= TDF_ASTPENDING; - } - if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) && - itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) { - p->p_sflag |= PS_PROFPEND; - td->td_flags |= TDF_ASTPENDING; - } + /* XXXKSE What to do? Should do more. */ } -#else +#endif +#endif pstats = p->p_stats; if (usermode && timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) && @@ -232,7 +221,6 @@ hardclock_cpu(int usermode) p->p_sflag |= PS_PROFPEND; td->td_flags |= TDF_ASTPENDING; } -#endif mtx_unlock_spin_flags(&sched_lock, MTX_QUIET); #ifdef HWPMC_HOOKS diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index 9c5597ec60c0..a12ed64ab41a 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -205,9 +205,6 @@ fork1(td, flags, pages, procp) struct filedesc *fd; struct filedesc_to_leader *fdtol; struct thread *td2; -#ifdef KSE - struct ksegrp *kg2; -#endif struct sigacts *newsigacts; int error; @@ -477,9 +474,6 @@ fork1(td, flags, pages, procp) * then copy the section that is copied directly from the parent. */ td2 = FIRST_THREAD_IN_PROC(p2); -#ifdef KSE - kg2 = FIRST_KSEGRP_IN_PROC(p2); -#endif /* Allocate and switch to an alternate kstack if specified. */ if (pages != 0) @@ -492,19 +486,11 @@ fork1(td, flags, pages, procp) __rangeof(struct proc, p_startzero, p_endzero)); bzero(&td2->td_startzero, __rangeof(struct thread, td_startzero, td_endzero)); -#ifdef KSE - bzero(&kg2->kg_startzero, - __rangeof(struct ksegrp, kg_startzero, kg_endzero)); -#endif bcopy(&p1->p_startcopy, &p2->p_startcopy, __rangeof(struct proc, p_startcopy, p_endcopy)); bcopy(&td->td_startcopy, &td2->td_startcopy, __rangeof(struct thread, td_startcopy, td_endcopy)); -#ifdef KSE - bcopy(&td->td_ksegrp->kg_startcopy, &kg2->kg_startcopy, - __rangeof(struct ksegrp, kg_startcopy, kg_endcopy)); -#endif td2->td_sigstk = td->td_sigstk; td2->td_sigmask = td->td_sigmask; @@ -526,11 +512,7 @@ fork1(td, flags, pages, procp) mtx_unlock_spin(&sched_lock); p2->p_ucred = crhold(td->td_ucred); -#ifdef KSE - td2->td_ucred = crhold(p2->p_ucred); /* XXXKSE */ -#else td2->td_ucred = crhold(p2->p_ucred); -#endif #ifdef AUDIT audit_proc_fork(p1, p2); #endif diff --git a/sys/kern/kern_idle.c b/sys/kern/kern_idle.c index f66056a26aa4..9096d985a36a 100644 --- a/sys/kern/kern_idle.c +++ b/sys/kern/kern_idle.c @@ -79,11 +79,7 @@ idle_setup(void *dummy) td = FIRST_THREAD_IN_PROC(p); TD_SET_CAN_RUN(td); td->td_flags |= TDF_IDLETD; -#ifdef KSE - sched_class(td->td_ksegrp, PRI_IDLE); -#else sched_class(td, PRI_IDLE); -#endif sched_prio(td, PRI_MAX_IDLE); mtx_unlock_spin(&sched_lock); PROC_UNLOCK(p); diff --git a/sys/kern/kern_intr.c b/sys/kern/kern_intr.c index 2aa4c47d4728..53cbc4b41076 100644 --- a/sys/kern/kern_intr.c +++ b/sys/kern/kern_intr.c @@ -296,11 +296,7 @@ ithread_create(const char *name) panic("kthread_create() failed with %d", error); td = FIRST_THREAD_IN_PROC(p); /* XXXKSE */ mtx_lock_spin(&sched_lock); -#ifdef KSE - td->td_ksegrp->kg_pri_class = PRI_ITHD; -#else - td->td_pri_class = PRI_ITHD; -#endif + sched_class(td, PRI_ITHD); TD_SET_IWAIT(td); mtx_unlock_spin(&sched_lock); td->td_pflags |= TDP_ITHREAD; diff --git a/sys/kern/kern_kse.c b/sys/kern/kern_kse.c index fadaa442fd77..4d27d71fec5e 100644 --- a/sys/kern/kern_kse.c +++ b/sys/kern/kern_kse.c @@ -48,9 +48,6 @@ __FBSDID("$FreeBSD$"); #include #ifdef KSE -/* - * KSEGRP related storage. - */ static uma_zone_t upcall_zone; /* DEBUG ONLY */ @@ -86,24 +83,24 @@ upcall_free(struct kse_upcall *ku) } void -upcall_link(struct kse_upcall *ku, struct ksegrp *kg) +upcall_link(struct kse_upcall *ku, struct proc *p) { mtx_assert(&sched_lock, MA_OWNED); - TAILQ_INSERT_TAIL(&kg->kg_upcalls, ku, ku_link); - ku->ku_ksegrp = kg; - kg->kg_numupcalls++; + TAILQ_INSERT_TAIL(&p->p_upcalls, ku, ku_link); + ku->ku_proc = p; + p->p_numupcalls++; } void upcall_unlink(struct kse_upcall *ku) { - struct ksegrp *kg = ku->ku_ksegrp; + struct proc *p = ku->ku_proc; mtx_assert(&sched_lock, MA_OWNED); KASSERT(ku->ku_owner == NULL, ("%s: have owner", __func__)); - TAILQ_REMOVE(&kg->kg_upcalls, ku, ku_link); - kg->kg_numupcalls--; + TAILQ_REMOVE(&p->p_upcalls, ku, ku_link); + p->p_numupcalls--; upcall_stash(ku); } @@ -305,7 +302,6 @@ kse_exit(struct thread *td, struct kse_exit_args *uap) { #ifdef KSE struct proc *p; - struct ksegrp *kg; struct kse_upcall *ku, *ku2; int error, count; @@ -316,11 +312,10 @@ kse_exit(struct thread *td, struct kse_exit_args *uap) if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) return (EINVAL); - kg = td->td_ksegrp; count = 0; /* - * Calculate the existing non-exiting upcalls in this ksegroup. + * Calculate the existing non-exiting upcalls in this process. * If we are the last upcall but there are still other threads, * then do not exit. We need the other threads to be able to * complete whatever they are doing. @@ -330,12 +325,12 @@ kse_exit(struct thread *td, struct kse_exit_args *uap) */ PROC_LOCK(p); mtx_lock_spin(&sched_lock); - FOREACH_UPCALL_IN_GROUP(kg, ku2) { + FOREACH_UPCALL_IN_PROC(p, ku2) { if (ku2->ku_flags & KUF_EXITING) count++; } - if ((kg->kg_numupcalls - count) == 1 && - (kg->kg_numthreads > 1)) { + if ((p->p_numupcalls - count) == 1 && + (p->p_numthreads > 1)) { mtx_unlock_spin(&sched_lock); PROC_UNLOCK(p); return (EDEADLK); @@ -360,20 +355,12 @@ kse_exit(struct thread *td, struct kse_exit_args *uap) mtx_lock_spin(&sched_lock); upcall_remove(td); if (p->p_numthreads != 1) { - /* - * If we are not the last thread, but we are the last - * thread in this ksegrp, then by definition this is not - * the last group and we need to clean it up as well. - * thread_exit will clean up the kseg as needed. - */ thread_stopped(p); thread_exit(); /* NOTREACHED */ } /* * This is the last thread. Just return to the user. - * We know that there is only one ksegrp too, as any others - * would have been discarded in previous calls to thread_exit(). * Effectively we have left threading mode.. * The only real thing left to do is ensure that the * scheduler sets out concurrency back to 1 as that may be a @@ -409,7 +396,6 @@ kse_release(struct thread *td, struct kse_release_args *uap) { #ifdef KSE struct proc *p; - struct ksegrp *kg; struct kse_upcall *ku; struct timespec timeout; struct timeval tv; @@ -417,7 +403,6 @@ kse_release(struct thread *td, struct kse_release_args *uap) int error; p = td->td_proc; - kg = td->td_ksegrp; if ((ku = td->td_upcall) == NULL || TD_CAN_UNBIND(td)) return (EINVAL); if (uap->timeout != NULL) { @@ -452,14 +437,14 @@ kse_release(struct thread *td, struct kse_release_args *uap) } else { if ((ku->ku_flags & KUF_DOUPCALL) == 0 && ((ku->ku_mflags & KMF_NOCOMPLETED) || - (kg->kg_completed == NULL))) { - kg->kg_upsleeps++; + (p->p_completed == NULL))) { + p->p_upsleeps++; td->td_kflags |= TDK_KSEREL; - error = msleep(&kg->kg_completed, &p->p_mtx, + error = msleep(&p->p_completed, &p->p_mtx, PPAUSE|PCATCH, "kserel", (uap->timeout ? tvtohz(&tv) : 0)); td->td_kflags &= ~(TDK_KSEREL | TDK_WAKEUP); - kg->kg_upsleeps--; + p->p_upsleeps--; } PROC_UNLOCK(p); } @@ -482,7 +467,6 @@ kse_wakeup(struct thread *td, struct kse_wakeup_args *uap) { #ifdef KSE struct proc *p; - struct ksegrp *kg; struct kse_upcall *ku; struct thread *td2; @@ -495,23 +479,18 @@ kse_wakeup(struct thread *td, struct kse_wakeup_args *uap) PROC_LOCK(p); mtx_lock_spin(&sched_lock); if (uap->mbx) { - FOREACH_KSEGRP_IN_PROC(p, kg) { - FOREACH_UPCALL_IN_GROUP(kg, ku) { - if (ku->ku_mailbox == uap->mbx) - break; - } - if (ku) + FOREACH_UPCALL_IN_PROC(p, ku) { + if (ku->ku_mailbox == uap->mbx) break; } } else { - kg = td->td_ksegrp; - if (kg->kg_upsleeps) { + if (p->p_upsleeps) { mtx_unlock_spin(&sched_lock); - wakeup(&kg->kg_completed); + wakeup(&p->p_completed); PROC_UNLOCK(p); return (0); } - ku = TAILQ_FIRST(&kg->kg_upcalls); + ku = TAILQ_FIRST(&p->p_upcalls); } if (ku == NULL) { mtx_unlock_spin(&sched_lock); @@ -526,7 +505,7 @@ kse_wakeup(struct thread *td, struct kse_wakeup_args *uap) if (!(td2->td_kflags & TDK_WAKEUP)) { td2->td_kflags |= TDK_WAKEUP; if (td2->td_kflags & TDK_KSEREL) - sleepq_remove(td2, &kg->kg_completed); + sleepq_remove(td2, &p->p_completed); else sleepq_remove(td2, &p->p_siglist); } @@ -542,11 +521,11 @@ kse_wakeup(struct thread *td, struct kse_wakeup_args *uap) } /* - * No new KSEG: first call: use current KSE, don't schedule an upcall + * newgroup == 0: first call: use current KSE, don't schedule an upcall * All other situations, do allocate max new KSEs and schedule an upcall. * * XXX should be changed so that 'first' behaviour lasts for as long - * as you have not made a kse in this ksegrp. i.e. as long as we do not have + * as you have not made a thread in this proc. i.e. as long as we do not have * a mailbox.. */ /* struct kse_create_args { @@ -557,8 +536,6 @@ int kse_create(struct thread *td, struct kse_create_args *uap) { #ifdef KSE - struct ksegrp *newkg; - struct ksegrp *kg; struct proc *p; struct kse_mailbox mbx; struct kse_upcall *newku; @@ -566,7 +543,21 @@ kse_create(struct thread *td, struct kse_create_args *uap) struct thread *newtd; p = td->td_proc; - kg = td->td_ksegrp; + + /* + * Processes using the other threading model can't + * suddenly start calling this one + * XXX maybe... + */ + if ((p->p_flag & (P_SA|P_HADTHREADS)) == P_HADTHREADS) { + PROC_UNLOCK(p); + return (EINVAL); + } + if (!(p->p_flag & P_SA)) { + first = 1; + p->p_flag |= P_SA|P_HADTHREADS; + } + if ((err = copyin(uap->mbx, &mbx, sizeof(mbx)))) return (err); @@ -577,123 +568,40 @@ kse_create(struct thread *td, struct kse_create_args *uap) * If the new UTS mailbox says that this * will be a BOUND lwp, then it had better * have its thread mailbox already there. - * In addition, this ksegrp will be limited to - * a concurrency of 1. There is more on this later. */ - if (mbx.km_flags & KMF_BOUND) { + if ((mbx.km_flags & KMF_BOUND) || uap->newgroup) { if (mbx.km_curthread == NULL) return (EINVAL); ncpus = 1; + if (!(uap->newgroup || first)) + return (EINVAL); } else { sa = TDP_SA; - } - - PROC_LOCK(p); - /* - * Processes using the other threading model can't - * suddenly start calling this one - */ - if ((p->p_flag & (P_SA|P_HADTHREADS)) == P_HADTHREADS) { - PROC_UNLOCK(p); - return (EINVAL); - } - - /* - * Limit it to NCPU upcall contexts per ksegrp in any case. - * There is a small race here as we don't hold proclock - * until we inc the ksegrp count, but it's not really a big problem - * if we get one too many, but we save a proc lock. - */ - if ((!uap->newgroup) && (kg->kg_numupcalls >= ncpus)) { - PROC_UNLOCK(p); - return (EPROCLIM); - } - - if (!(p->p_flag & P_SA)) { - first = 1; - p->p_flag |= P_SA|P_HADTHREADS; - } - - PROC_UNLOCK(p); - /* - * Now pay attention! - * If we are going to be bound, then we need to be either - * a new group, or the first call ever. In either - * case we will be creating (or be) the only thread in a group. - * and the concurrency will be set to 1. - * This is not quite right, as we may still make ourself - * bound after making other ksegrps but it will do for now. - * The library will only try do this much. - */ - if (!sa && !(uap->newgroup || first)) - return (EINVAL); - - if (uap->newgroup) { - newkg = ksegrp_alloc(); - bzero(&newkg->kg_startzero, - __rangeof(struct ksegrp, kg_startzero, kg_endzero)); - bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, - __rangeof(struct ksegrp, kg_startcopy, kg_endcopy)); - sched_init_concurrency(newkg); PROC_LOCK(p); - if (p->p_numksegrps >= max_groups_per_proc) { + /* + * Limit it to NCPU upcall contexts per proc in any case. + */ + if (p->p_numupcalls >= ncpus) { PROC_UNLOCK(p); - ksegrp_free(newkg); return (EPROCLIM); } - ksegrp_link(newkg, p); - mtx_lock_spin(&sched_lock); - sched_fork_ksegrp(td, newkg); - mtx_unlock_spin(&sched_lock); - PROC_UNLOCK(p); - } else { /* - * We want to make a thread in our own ksegrp. + * We want to make a thread (bound or unbound). * If we are just the first call, either kind * is ok, but if not then either we must be * already an upcallable thread to make another, * or a bound thread to make one of those. * Once again, not quite right but good enough for now.. XXXKSE + * XXX bogus */ + PROC_UNLOCK(p); if (!first && ((td->td_pflags & TDP_SA) != sa)) return (EINVAL); - - newkg = kg; + if (p->p_numupcalls == 0) { + sched_set_concurrency(p, ncpus); + } } - /* - * This test is a bit "indirect". - * It might simplify things if we made a direct way of testing - * if a ksegrp has been worked on before. - * In the case of a bound request and the concurrency being set to - * one, the concurrency will already be 1 so it's just inefficient - * but not dangerous to call this again. XXX - */ - if (newkg->kg_numupcalls == 0) { - /* - * Initialize KSE group with the appropriate - * concurrency. - * - * For a multiplexed group, create as as much concurrency - * as the number of physical cpus. - * This increases concurrency in the kernel even if the - * userland is not MP safe and can only run on a single CPU. - * In an ideal world, every physical cpu should execute a - * thread. If there is enough concurrency, threads in the - * kernel can be executed parallel on different cpus at - * full speed without being restricted by the number of - * upcalls the userland provides. - * Adding more upcall structures only increases concurrency - * in userland. - * - * For a bound thread group, because there is only one thread - * in the group, we only set the concurrency for the group - * to 1. A thread in this kind of group will never schedule - * an upcall when blocked. This simulates pthread system - * scope thread behaviour. - */ - sched_set_concurrency(newkg, ncpus); - } /* * Even bound LWPs get a mailbox and an upcall to hold it. */ @@ -711,33 +619,38 @@ kse_create(struct thread *td, struct kse_create_args *uap) PROC_LOCK(p); mtx_lock_spin(&sched_lock); - if (newkg->kg_numupcalls >= ncpus) { - mtx_unlock_spin(&sched_lock); - PROC_UNLOCK(p); - upcall_free(newku); - return (EPROCLIM); + if (sa) { + if( p->p_numupcalls >= ncpus) { + mtx_unlock_spin(&sched_lock); + PROC_UNLOCK(p); + upcall_free(newku); + return (EPROCLIM); + } + + /* + * If we are the first time, and a normal thread, + * then transfer all the signals back to the 'process'. + * SA threading will make a special thread to handle them. + */ + if (first) { + sigqueue_move_set(&td->td_sigqueue, &p->p_sigqueue, + &td->td_sigqueue.sq_signals); + SIGFILLSET(td->td_sigmask); + SIG_CANTMASK(td->td_sigmask); + } + } else { + /* should subtract from process count (later) */ } /* - * If we are the first time, and a normal thread, - * then transfer all the signals back to the 'process'. - * SA threading will make a special thread to handle them. - */ - if (first && sa) { - sigqueue_move_set(&td->td_sigqueue, &p->p_sigqueue, - &td->td_sigqueue.sq_signals); - SIGFILLSET(td->td_sigmask); - SIG_CANTMASK(td->td_sigmask); - } - - /* - * Make the new upcall available to the ksegrp. + * Make the new upcall available to the process. * It may or may not use it, but it's available. */ - upcall_link(newku, newkg); + upcall_link(newku, p); PROC_UNLOCK(p); if (mbx.km_quantum) - newkg->kg_upquantum = max(1, mbx.km_quantum / tick); +/* XXX should this be in the thread? */ + p->p_upquantum = max(1, mbx.km_quantum / tick); /* * Each upcall structure has an owner thread, find which @@ -745,8 +658,11 @@ kse_create(struct thread *td, struct kse_create_args *uap) */ if (uap->newgroup) { /* - * Because the new ksegrp hasn't a thread, - * create an initial upcall thread to own it. + * The newgroup parameter now means + * "bound, non SA, system scope" + * It is only used for the interrupt thread at the + * moment I think + * We'll rename it later. */ newtd = thread_schedule_upcall(td, newku); } else { @@ -771,6 +687,7 @@ kse_create(struct thread *td, struct kse_create_args *uap) /* * Let the UTS instance know its LWPID. * It doesn't really care. But the debugger will. + * XXX warning.. remember that this moves. */ suword32(&newku->ku_mailbox->km_lwp, newtd->td_tid); @@ -785,6 +702,14 @@ kse_create(struct thread *td, struct kse_create_args *uap) if (sa) { newtd->td_pflags |= TDP_SA; + /* + * If we are starting a new thread, kick it off. + */ + if (newtd != td) { + mtx_lock_spin(&sched_lock); + setrunqueue(newtd, SRQ_BORING); + mtx_unlock_spin(&sched_lock); + } } else { newtd->td_pflags &= ~TDP_SA; @@ -816,17 +741,11 @@ kse_create(struct thread *td, struct kse_create_args *uap) _PRELE(p); } PROC_UNLOCK(p); + mtx_lock_spin(&sched_lock); + setrunqueue(newtd, SRQ_BORING); + mtx_unlock_spin(&sched_lock); } } - - /* - * If we are starting a new thread, kick it off. - */ - if (newtd != td) { - mtx_lock_spin(&sched_lock); - setrunqueue(newtd, SRQ_BORING); - mtx_unlock_spin(&sched_lock); - } return (0); #else /* !KSE */ return (EOPNOTSUPP); @@ -886,20 +805,18 @@ kse_GC(void) /* * Store the thread context in the UTS's mailbox. * then add the mailbox at the head of a list we are building in user space. - * The list is anchored in the ksegrp structure. + * The list is anchored in the proc structure. */ int thread_export_context(struct thread *td, int willexit) { struct proc *p; - struct ksegrp *kg; uintptr_t mbx; void *addr; int error = 0, sig; mcontext_t mc; p = td->td_proc; - kg = td->td_ksegrp; /* * Post sync signal, or process SIGKILL and SIGSTOP. @@ -940,14 +857,14 @@ thread_export_context(struct thread *td, int willexit) * entry into this one */ for (;;) { - mbx = (uintptr_t)kg->kg_completed; + mbx = (uintptr_t)p->p_completed; if (suword(addr, mbx)) { error = EFAULT; goto bad; } PROC_LOCK(p); - if (mbx == (uintptr_t)kg->kg_completed) { - kg->kg_completed = td->td_mailbox; + if (mbx == (uintptr_t)p->p_completed) { + p->p_completed = td->td_mailbox; /* * The thread context may be taken away by * other upcall threads when we unlock @@ -970,19 +887,18 @@ thread_export_context(struct thread *td, int willexit) } /* - * Take the list of completed mailboxes for this KSEGRP and put them on this + * Take the list of completed mailboxes for this Process and put them on this * upcall's mailbox as it's the next one going up. */ static int -thread_link_mboxes(struct ksegrp *kg, struct kse_upcall *ku) +thread_link_mboxes(struct proc *p, struct kse_upcall *ku) { - struct proc *p = kg->kg_proc; void *addr; uintptr_t mbx; addr = (void *)(&ku->ku_mailbox->km_completed); for (;;) { - mbx = (uintptr_t)kg->kg_completed; + mbx = (uintptr_t)p->p_completed; if (suword(addr, mbx)) { PROC_LOCK(p); psignal(p, SIGSEGV); @@ -990,8 +906,8 @@ thread_link_mboxes(struct ksegrp *kg, struct kse_upcall *ku) return (EFAULT); } PROC_LOCK(p); - if (mbx == (uintptr_t)kg->kg_completed) { - kg->kg_completed = NULL; + if (mbx == (uintptr_t)p->p_completed) { + p->p_completed = NULL; PROC_UNLOCK(p); break; } @@ -1109,7 +1025,7 @@ thread_schedule_upcall(struct thread *td, struct kse_upcall *ku) */ bcopy(&td->td_startcopy, &td2->td_startcopy, __rangeof(struct thread, td_startcopy, td_endcopy)); - thread_link(td2, ku->ku_ksegrp); + thread_link(td2, ku->ku_proc); /* inherit parts of blocked thread's context as a good template */ cpu_set_upcall(td2, td); /* Let the new thread become owner of the upcall */ @@ -1210,7 +1126,6 @@ void thread_user_enter(struct thread *td) { struct proc *p = td->td_proc; - struct ksegrp *kg; struct kse_upcall *ku; struct kse_thr_mailbox *tmbx; uint32_t flags; @@ -1233,7 +1148,6 @@ thread_user_enter(struct thread *td) * note where our mailbox is. */ - kg = td->td_ksegrp; ku = td->td_upcall; KASSERT(ku != NULL, ("no upcall owned")); @@ -1291,10 +1205,9 @@ int thread_userret(struct thread *td, struct trapframe *frame) { struct kse_upcall *ku; - struct ksegrp *kg, *kg2; struct proc *p; struct timespec ts; - int error = 0, upcalls, uts_crit; + int error = 0, uts_crit; /* Nothing to do with bound thread */ if (!(td->td_pflags & TDP_SA)) @@ -1311,7 +1224,6 @@ thread_userret(struct thread *td, struct trapframe *frame) } p = td->td_proc; - kg = td->td_ksegrp; ku = td->td_upcall; /* @@ -1323,9 +1235,9 @@ thread_userret(struct thread *td, struct trapframe *frame) if (TD_CAN_UNBIND(td)) { td->td_pflags &= ~TDP_CAN_UNBIND; if ((td->td_flags & TDF_NEEDSIGCHK) == 0 && - (kg->kg_completed == NULL) && + (p->p_completed == NULL) && (ku->ku_flags & KUF_DOUPCALL) == 0 && - (kg->kg_upquantum && ticks < kg->kg_nextupcall)) { + (p->p_upquantum && ticks < p->p_nextupcall)) { nanotime(&ts); error = copyout(&ts, (caddr_t)&ku->ku_mailbox->km_timeofday, @@ -1346,8 +1258,8 @@ thread_userret(struct thread *td, struct trapframe *frame) } else if (td->td_mailbox && (ku == NULL)) { thread_export_context(td, 1); PROC_LOCK(p); - if (kg->kg_upsleeps) - wakeup(&kg->kg_completed); + if (p->p_upsleeps) + wakeup(&p->p_completed); WITNESS_WARN(WARN_PANIC, &p->p_mtx.mtx_object, "thread exiting in userret"); sigqueue_flush(&td->td_sigqueue); @@ -1366,14 +1278,7 @@ thread_userret(struct thread *td, struct trapframe *frame) mtx_lock_spin(&sched_lock); p->p_maxthrwaits++; while (p->p_numthreads > max_threads_per_proc) { - upcalls = 0; - FOREACH_KSEGRP_IN_PROC(p, kg2) { - if (kg2->kg_numupcalls == 0) - upcalls++; - else - upcalls += kg2->kg_numupcalls; - } - if (upcalls >= max_threads_per_proc) + if (p->p_numupcalls >= max_threads_per_proc) break; mtx_unlock_spin(&sched_lock); if (msleep(&p->p_numthreads, &p->p_mtx, PPAUSE|PCATCH, @@ -1391,7 +1296,7 @@ thread_userret(struct thread *td, struct trapframe *frame) if (td->td_pflags & TDP_UPCALLING) { uts_crit = 0; - kg->kg_nextupcall = ticks + kg->kg_upquantum; + p->p_nextupcall = ticks + p->p_upquantum; /* * There is no more work to do and we are going to ride * this thread up to userland as an upcall. @@ -1436,7 +1341,7 @@ thread_userret(struct thread *td, struct trapframe *frame) * this KSE's mailbox. */ if (!(ku->ku_mflags & KMF_NOCOMPLETED) && - (error = thread_link_mboxes(kg, ku)) != 0) + (error = thread_link_mboxes(p, ku)) != 0) goto out; } if (!uts_crit) { @@ -1479,7 +1384,6 @@ thread_userret(struct thread *td, struct trapframe *frame) void thread_continued(struct proc *p) { - struct ksegrp *kg; struct kse_upcall *ku; struct thread *td; @@ -1490,18 +1394,13 @@ thread_continued(struct proc *p) return; if (p->p_flag & P_TRACED) { - FOREACH_KSEGRP_IN_PROC(p, kg) { - td = TAILQ_FIRST(&kg->kg_threads); - if (td == NULL) - continue; - /* not a SA group, nothing to do */ - if (!(td->td_pflags & TDP_SA)) - continue; - FOREACH_UPCALL_IN_GROUP(kg, ku) { + td = TAILQ_FIRST(&p->p_threads); + if (td && (td->td_pflags & TDP_SA)) { + FOREACH_UPCALL_IN_PROC(p, ku) { mtx_lock_spin(&sched_lock); ku->ku_flags |= KUF_DOUPCALL; mtx_unlock_spin(&sched_lock); - wakeup(&kg->kg_completed); + wakeup(&p->p_completed); } } } diff --git a/sys/kern/kern_poll.c b/sys/kern/kern_poll.c index 67338b06d71e..8e87607733ef 100644 --- a/sys/kern/kern_poll.c +++ b/sys/kern/kern_poll.c @@ -581,11 +581,7 @@ poll_idle(void) rtp.prio = RTP_PRIO_MAX; /* lowest priority */ rtp.type = RTP_PRIO_IDLE; mtx_lock_spin(&sched_lock); -#ifdef KSE - rtp_to_pri(&rtp, td->td_ksegrp); -#else rtp_to_pri(&rtp, td); -#endif mtx_unlock_spin(&sched_lock); for (;;) { diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c index 2401b6b58441..0eb44076d32b 100644 --- a/sys/kern/kern_proc.c +++ b/sys/kern/kern_proc.c @@ -141,9 +141,6 @@ proc_dtor(void *mem, int size, void *arg) { struct proc *p; struct thread *td; -#if defined(INVARIANTS) && defined(KSE) - struct ksegrp *kg; -#endif /* INVARIANTS checks go here */ p = (struct proc *)mem; @@ -151,14 +148,7 @@ proc_dtor(void *mem, int size, void *arg) #ifdef INVARIANTS KASSERT((p->p_numthreads == 1), ("bad number of threads in exiting process")); -#ifdef KSE - KASSERT((p->p_numksegrps == 1), ("free proc with > 1 ksegrp")); -#endif KASSERT((td != NULL), ("proc_dtor: bad thread pointer")); -#ifdef KSE - kg = FIRST_KSEGRP_IN_PROC(p); - KASSERT((kg != NULL), ("proc_dtor: bad kg pointer")); -#endif KASSERT(STAILQ_EMPTY(&p->p_ktr), ("proc_dtor: non-empty p_ktr")); #endif @@ -181,25 +171,15 @@ proc_init(void *mem, int size, int flags) { struct proc *p; struct thread *td; -#ifdef KSE - struct ksegrp *kg; -#endif p = (struct proc *)mem; p->p_sched = (struct p_sched *)&p[1]; td = thread_alloc(); -#ifdef KSE - kg = ksegrp_alloc(); -#endif bzero(&p->p_mtx, sizeof(struct mtx)); mtx_init(&p->p_mtx, "process lock", NULL, MTX_DEF | MTX_DUPOK); p->p_stats = pstats_alloc(); -#ifdef KSE - proc_linkup(p, kg, td); - sched_newproc(p, kg, td); -#else proc_linkup(p, td); -#endif + sched_newproc(p, td); return (0); } @@ -215,9 +195,6 @@ proc_fini(void *mem, int size) p = (struct proc *)mem; pstats_free(p->p_stats); -#ifdef KSE - ksegrp_free(FIRST_KSEGRP_IN_PROC(p)); -#endif thread_free(FIRST_THREAD_IN_PROC(p)); mtx_destroy(&p->p_mtx); if (p->p_ksi != NULL) @@ -782,9 +759,6 @@ fill_kinfo_proc_only(struct proc *p, struct kinfo_proc *kp) static void fill_kinfo_thread(struct thread *td, struct kinfo_proc *kp) { -#ifdef KSE - struct ksegrp *kg; -#endif struct proc *p; p = td->td_proc; @@ -824,15 +798,6 @@ fill_kinfo_thread(struct thread *td, struct kinfo_proc *kp) kp->ki_stat = SIDL; } -#ifdef KSE - kg = td->td_ksegrp; - - /* things in the KSE GROUP */ - kp->ki_estcpu = kg->kg_estcpu; - kp->ki_slptime = kg->kg_slptime; - kp->ki_pri.pri_user = kg->kg_user_pri; - kp->ki_pri.pri_class = kg->kg_pri_class; -#endif /* Things in the thread */ kp->ki_wchan = td->td_wchan; kp->ki_pri.pri_level = td->td_priority; @@ -845,12 +810,10 @@ fill_kinfo_thread(struct thread *td, struct kinfo_proc *kp) kp->ki_pcb = td->td_pcb; kp->ki_kstack = (void *)td->td_kstack; kp->ki_pctcpu = sched_pctcpu(td); -#ifndef KSE kp->ki_estcpu = td->td_estcpu; kp->ki_slptime = td->td_slptime; kp->ki_pri.pri_class = td->td_pri_class; kp->ki_pri.pri_user = td->td_user_pri; -#endif /* We can't get this anymore but ps etc never used it anyway. */ kp->ki_rqindex = 0; diff --git a/sys/kern/kern_resource.c b/sys/kern/kern_resource.c index f885953d4b16..e351d18cc2d7 100644 --- a/sys/kern/kern_resource.c +++ b/sys/kern/kern_resource.c @@ -319,11 +319,7 @@ rtprio_thread(struct thread *td, struct rtprio_thread_args *uap) else td1 = thread_find(p, uap->lwpid); if (td1 != NULL) -#ifdef KSE - pri_to_rtp(td1->td_ksegrp, &rtp); -#else pri_to_rtp(td1, &rtp); -#endif else error = ESRCH; mtx_unlock_spin(&sched_lock); @@ -359,11 +355,7 @@ rtprio_thread(struct thread *td, struct rtprio_thread_args *uap) else td1 = thread_find(p, uap->lwpid); if (td1 != NULL) -#ifdef KSE - error = rtp_to_pri(&rtp, td1->td_ksegrp); -#else error = rtp_to_pri(&rtp, td1); -#endif else error = ESRCH; mtx_unlock_spin(&sched_lock); @@ -396,11 +388,7 @@ rtprio(td, uap) { struct proc *curp; struct proc *p; -#ifdef KSE - struct ksegrp *kg; -#else struct thread *tdp; -#endif struct rtprio rtp; int cierror, error; @@ -436,23 +424,14 @@ rtprio(td, uap) * as leaving it zero. */ if (uap->pid == 0) { -#ifdef KSE - pri_to_rtp(td->td_ksegrp, &rtp); -#else pri_to_rtp(td, &rtp); -#endif } else { struct rtprio rtp2; rtp.type = RTP_PRIO_IDLE; rtp.prio = RTP_PRIO_MAX; -#ifdef KSE - FOREACH_KSEGRP_IN_PROC(p, kg) { - pri_to_rtp(kg, &rtp2); -#else FOREACH_THREAD_IN_PROC(p, tdp) { pri_to_rtp(tdp, &rtp2); -#endif if (rtp2.type < rtp.type || (rtp2.type == rtp.type && rtp2.prio < rtp.prio)) { @@ -493,39 +472,19 @@ rtprio(td, uap) } } -#ifdef KSE - /* - * If we are setting our own priority, set just our - * KSEGRP but if we are doing another process, - * do all the groups on that process. If we - * specify our own pid we do the latter. - */ -#else /* * If we are setting our own priority, set just our * thread but if we are doing another process, * do all the threads on that process. If we * specify our own pid we do the latter. */ -#endif mtx_lock_spin(&sched_lock); if (uap->pid == 0) { -#ifdef KSE - error = rtp_to_pri(&rtp, td->td_ksegrp); -#else error = rtp_to_pri(&rtp, td); -#endif } else { -#ifdef KSE - FOREACH_KSEGRP_IN_PROC(p, kg) { - if ((error = rtp_to_pri(&rtp, kg)) != 0) { - break; - } -#else FOREACH_THREAD_IN_PROC(p, td) { if ((error = rtp_to_pri(&rtp, td)) != 0) break; -#endif } } mtx_unlock_spin(&sched_lock); @@ -539,11 +498,7 @@ rtprio(td, uap) } int -#ifdef KSE -rtp_to_pri(struct rtprio *rtp, struct ksegrp *kg) -#else rtp_to_pri(struct rtprio *rtp, struct thread *td) -#endif { u_char newpri; @@ -552,87 +507,43 @@ rtp_to_pri(struct rtprio *rtp, struct thread *td) return (EINVAL); switch (RTP_PRIO_BASE(rtp->type)) { case RTP_PRIO_REALTIME: -#ifdef KSE newpri = PRI_MIN_REALTIME + rtp->prio; -#else - newpri = PRI_MIN_REALTIME + rtp->prio; -#endif break; case RTP_PRIO_NORMAL: -#ifdef KSE newpri = PRI_MIN_TIMESHARE + rtp->prio; -#else - newpri = PRI_MIN_TIMESHARE + rtp->prio; -#endif break; case RTP_PRIO_IDLE: -#ifdef KSE newpri = PRI_MIN_IDLE + rtp->prio; -#else - newpri = PRI_MIN_IDLE + rtp->prio; -#endif break; default: return (EINVAL); } -#ifdef KSE - sched_class(kg, rtp->type); - sched_user_prio(kg, newpri); - if (curthread->td_ksegrp == kg) { - sched_prio(curthread, kg->kg_user_pri); /* XXX dubious */ - } -#else sched_class(td, rtp->type); /* XXX fix */ sched_user_prio(td, newpri); if (curthread == td) sched_prio(curthread, td->td_user_pri); /* XXX dubious */ -#endif return (0); } void -#ifdef KSE -pri_to_rtp(struct ksegrp *kg, struct rtprio *rtp) -#else pri_to_rtp(struct thread *td, struct rtprio *rtp) -#endif { mtx_assert(&sched_lock, MA_OWNED); -#ifdef KSE - switch (PRI_BASE(kg->kg_pri_class)) { -#else switch (PRI_BASE(td->td_pri_class)) { -#endif case PRI_REALTIME: -#ifdef KSE - rtp->prio = kg->kg_base_user_pri - PRI_MIN_REALTIME; -#else rtp->prio = td->td_base_user_pri - PRI_MIN_REALTIME; -#endif break; case PRI_TIMESHARE: -#ifdef KSE - rtp->prio = kg->kg_base_user_pri - PRI_MIN_TIMESHARE; -#else rtp->prio = td->td_base_user_pri - PRI_MIN_TIMESHARE; -#endif break; case PRI_IDLE: -#ifdef KSE - rtp->prio = kg->kg_base_user_pri - PRI_MIN_IDLE; -#else rtp->prio = td->td_base_user_pri - PRI_MIN_IDLE; -#endif break; default: break; } -#ifdef KSE - rtp->type = kg->kg_pri_class; -#else rtp->type = td->td_pri_class; -#endif } #if defined(COMPAT_43) diff --git a/sys/kern/kern_subr.c b/sys/kern/kern_subr.c index 631dfa2748df..ba288acfcdfc 100644 --- a/sys/kern/kern_subr.c +++ b/sys/kern/kern_subr.c @@ -430,11 +430,7 @@ uio_yield(void) td = curthread; mtx_lock_spin(&sched_lock); DROP_GIANT(); -#ifdef KSE - sched_prio(td, td->td_ksegrp->kg_user_pri); /* XXXKSE */ -#else sched_prio(td, td->td_user_pri); -#endif mi_switch(SW_INVOL, NULL); mtx_unlock_spin(&sched_lock); PICKUP_GIANT(); diff --git a/sys/kern/kern_switch.c b/sys/kern/kern_switch.c index 152114320341..039c7811b418 100644 --- a/sys/kern/kern_switch.c +++ b/sys/kern/kern_switch.c @@ -24,68 +24,6 @@ * SUCH DAMAGE. */ -#ifdef KSE -/*** -Here is the logic.. - -If there are N processors, then there are at most N KSEs (kernel -schedulable entities) working to process threads that belong to a -KSEGROUP (kg). If there are X of these KSEs actually running at the -moment in question, then there are at most M (N-X) of these KSEs on -the run queue, as running KSEs are not on the queue. - -Runnable threads are queued off the KSEGROUP in priority order. -If there are M or more threads runnable, the top M threads -(by priority) are 'preassigned' to the M KSEs not running. The KSEs take -their priority from those threads and are put on the run queue. - -The last thread that had a priority high enough to have a KSE associated -with it, AND IS ON THE RUN QUEUE is pointed to by -kg->kg_last_assigned. If no threads queued off the KSEGROUP have KSEs -assigned as all the available KSEs are activly running, or because there -are no threads queued, that pointer is NULL. - -When a KSE is removed from the run queue to become runnable, we know -it was associated with the highest priority thread in the queue (at the head -of the queue). If it is also the last assigned we know M was 1 and must -now be 0. Since the thread is no longer queued that pointer must be -removed from it. Since we know there were no more KSEs available, -(M was 1 and is now 0) and since we are not FREEING our KSE -but using it, we know there are STILL no more KSEs available, we can prove -that the next thread in the ksegrp list will not have a KSE to assign to -it, so we can show that the pointer must be made 'invalid' (NULL). - -The pointer exists so that when a new thread is made runnable, it can -have its priority compared with the last assigned thread to see if -it should 'steal' its KSE or not.. i.e. is it 'earlier' -on the list than that thread or later.. If it's earlier, then the KSE is -removed from the last assigned (which is now not assigned a KSE) -and reassigned to the new thread, which is placed earlier in the list. -The pointer is then backed up to the previous thread (which may or may not -be the new thread). - -When a thread sleeps or is removed, the KSE becomes available and if there -are queued threads that are not assigned KSEs, the highest priority one of -them is assigned the KSE, which is then placed back on the run queue at -the approipriate place, and the kg->kg_last_assigned pointer is adjusted down -to point to it. - -The following diagram shows 2 KSEs and 3 threads from a single process. - - RUNQ: --->KSE---KSE--... (KSEs queued at priorities from threads) - \ \____ - \ \ - KSEGROUP---thread--thread--thread (queued in priority order) - \ / - \_______________/ - (last_assigned) - -The result of this scheme is that the M available KSEs are always -queued at the priorities they have inherrited from the M highest priority -threads for that KSEGROUP. If this situation changes, the KSEs are -reassigned to keep this true. -***/ -#endif #include __FBSDID("$FreeBSD$"); @@ -126,8 +64,6 @@ __FBSDID("$FreeBSD$"); CTASSERT((RQB_BPW * RQB_LEN) == RQ_NQS); -#define td_kse td_sched - /* * kern.sched.preemption allows user space to determine if preemption support * is compiled in or not. It is not currently a boot or runtime flag that @@ -144,79 +80,40 @@ SYSCTL_INT(_kern_sched, OID_AUTO, preemption, CTLFLAG_RD, /************************************************************************ * Functions that manipulate runnability from a thread perspective. * ************************************************************************/ -#ifdef KSE -/* - * Select the KSE that will be run next. From that find the thread, and - * remove it from the KSEGRP's run queue. If there is thread clustering, - * this will be what does it. - */ -#else /* * Select the thread that will be run next. */ -#endif struct thread * choosethread(void) { -#ifdef KSE - struct kse *ke; -#endif + struct td_sched *ts; struct thread *td; -#ifdef KSE - struct ksegrp *kg; -#endif #if defined(SMP) && (defined(__i386__) || defined(__amd64__)) if (smp_active == 0 && PCPU_GET(cpuid) != 0) { /* Shutting down, run idlethread on AP's */ td = PCPU_GET(idlethread); -#ifdef KSE - ke = td->td_kse; -#endif + ts = td->td_sched; CTR1(KTR_RUNQ, "choosethread: td=%p (idle)", td); -#ifdef KSE - ke->ke_flags |= KEF_DIDRUN; -#else - td->td_kse->ke_flags |= KEF_DIDRUN; -#endif + ts->ts_flags |= TSF_DIDRUN; TD_SET_RUNNING(td); return (td); } #endif retry: -#ifdef KSE - ke = sched_choose(); - if (ke) { - td = ke->ke_thread; - KASSERT((td->td_kse == ke), ("kse/thread mismatch")); - kg = ke->ke_ksegrp; - if (td->td_proc->p_flag & P_HADTHREADS) { - if (kg->kg_last_assigned == td) { - kg->kg_last_assigned = TAILQ_PREV(td, - threadqueue, td_runq); - } - TAILQ_REMOVE(&kg->kg_runq, td, td_runq); - } -#else - td = sched_choose(); - if (td) { -#endif + ts = sched_choose(); + if (ts) { + td = ts->ts_thread; CTR2(KTR_RUNQ, "choosethread: td=%p pri=%d", td, td->td_priority); } else { /* Simulate runq_choose() having returned the idle thread */ td = PCPU_GET(idlethread); -#ifdef KSE - ke = td->td_kse; -#endif + ts = td->td_sched; CTR1(KTR_RUNQ, "choosethread: td=%p (idle)", td); } -#ifdef KSE - ke->ke_flags |= KEF_DIDRUN; -#else - td->td_kse->ke_flags |= KEF_DIDRUN; -#endif + ts->ts_flags |= TSF_DIDRUN; /* * If we are in panic, only allow system threads, @@ -233,91 +130,24 @@ choosethread(void) return (td); } -#ifdef KSE + +#if 0 /* - * Given a surplus system slot, try assign a new runnable thread to it. - * Called from: - * sched_thread_exit() (local) - * sched_switch() (local) - * sched_thread_exit() (local) - * remrunqueue() (local) (not at the moment) - */ -static void -slot_fill(struct ksegrp *kg) -{ - struct thread *td; - - mtx_assert(&sched_lock, MA_OWNED); - while (kg->kg_avail_opennings > 0) { - /* - * Find the first unassigned thread - */ - if ((td = kg->kg_last_assigned) != NULL) - td = TAILQ_NEXT(td, td_runq); - else - td = TAILQ_FIRST(&kg->kg_runq); - - /* - * If we found one, send it to the system scheduler. - */ - if (td) { - kg->kg_last_assigned = td; - sched_add(td, SRQ_YIELDING); - CTR2(KTR_RUNQ, "slot_fill: td%p -> kg%p", td, kg); - } else { - /* no threads to use up the slots. quit now */ - break; - } - } -} - -#ifdef SCHED_4BSD -/* - * Remove a thread from its KSEGRP's run queue. - * This in turn may remove it from a KSE if it was already assigned - * to one, possibly causing a new thread to be assigned to the KSE - * and the KSE getting a new priority. + * currently not used.. threads remove themselves from the + * run queue by running. */ static void remrunqueue(struct thread *td) { - struct thread *td2, *td3; - struct ksegrp *kg; - struct kse *ke; - mtx_assert(&sched_lock, MA_OWNED); KASSERT((TD_ON_RUNQ(td)), ("remrunqueue: Bad state on run queue")); - kg = td->td_ksegrp; - ke = td->td_kse; CTR1(KTR_RUNQ, "remrunqueue: td%p", td); TD_SET_CAN_RUN(td); - /* - * If it is not a threaded process, take the shortcut. - */ - if ((td->td_proc->p_flag & P_HADTHREADS) == 0) { - /* remve from sys run queue and free up a slot */ - sched_rem(td); - return; - } - td3 = TAILQ_PREV(td, threadqueue, td_runq); - TAILQ_REMOVE(&kg->kg_runq, td, td_runq); - if (ke->ke_state == KES_ONRUNQ) { - /* - * This thread has been assigned to the system run queue. - * We need to dissociate it and try assign the - * KSE to the next available thread. Then, we should - * see if we need to move the KSE in the run queues. - */ - sched_rem(td); - td2 = kg->kg_last_assigned; - KASSERT((td2 != NULL), ("last assigned has wrong value")); - if (td2 == td) - kg->kg_last_assigned = td3; - /* slot_fill(kg); */ /* will replace it with another */ - } + /* remove from sys run queue */ + sched_rem(td); + return; } #endif -#endif /* * Change the priority of a thread that is on the run queue. @@ -325,229 +155,32 @@ remrunqueue(struct thread *td) void adjustrunqueue( struct thread *td, int newpri) { -#ifdef KSE - struct ksegrp *kg; -#endif - struct kse *ke; + struct td_sched *ts; mtx_assert(&sched_lock, MA_OWNED); KASSERT((TD_ON_RUNQ(td)), ("adjustrunqueue: Bad state on run queue")); - ke = td->td_kse; + ts = td->td_sched; CTR1(KTR_RUNQ, "adjustrunqueue: td%p", td); -#ifdef KSE - /* - * If it is not a threaded process, take the shortcut. - */ - if ((td->td_proc->p_flag & P_HADTHREADS) == 0) { - /* We only care about the kse in the run queue. */ - td->td_priority = newpri; -#ifndef SCHED_CORE - if (ke->ke_rqindex != (newpri / RQ_PPQ)) -#else - if (ke->ke_rqindex != newpri) -#endif - { - sched_rem(td); - sched_add(td, SRQ_BORING); - } - return; - } - - /* It is a threaded process */ - kg = td->td_ksegrp; - if (ke->ke_state == KES_ONRUNQ -#ifdef SCHED_ULE - || ((ke->ke_flags & KEF_ASSIGNED) != 0 && - (ke->ke_flags & KEF_REMOVED) == 0) -#endif - ) { - if (kg->kg_last_assigned == td) { - kg->kg_last_assigned = - TAILQ_PREV(td, threadqueue, td_runq); - } - sched_rem(td); - } - TAILQ_REMOVE(&kg->kg_runq, td, td_runq); - TD_SET_CAN_RUN(td); - td->td_priority = newpri; - setrunqueue(td, SRQ_BORING); -#else - /* We only care about the kse in the run queue. */ + /* We only care about the td_sched in the run queue. */ td->td_priority = newpri; #ifndef SCHED_CORE - if (ke->ke_rqindex != (newpri / RQ_PPQ)) + if (ts->ts_rqindex != (newpri / RQ_PPQ)) #else - if (ke->ke_rqindex != newpri) + if (ts->ts_rqindex != newpri) #endif { sched_rem(td); sched_add(td, SRQ_BORING); } -#endif } -#ifdef KSE -/* - * This function is called when a thread is about to be put on a - * ksegrp run queue because it has been made runnable or its - * priority has been adjusted and the ksegrp does not have a - * free kse slot. It determines if a thread from the same ksegrp - * should be preempted. If so, it tries to switch threads - * if the thread is on the same cpu or notifies another cpu that - * it should switch threads. - */ - -static void -maybe_preempt_in_ksegrp(struct thread *td) -#if !defined(SMP) -{ - struct thread *running_thread; - - mtx_assert(&sched_lock, MA_OWNED); - running_thread = curthread; - - if (running_thread->td_ksegrp != td->td_ksegrp) - return; - - if (td->td_priority >= running_thread->td_priority) - return; -#ifdef PREEMPTION -#ifndef FULL_PREEMPTION - if (td->td_priority > PRI_MAX_ITHD) { - running_thread->td_flags |= TDF_NEEDRESCHED; - return; - } -#endif /* FULL_PREEMPTION */ - - if (running_thread->td_critnest > 1) - running_thread->td_owepreempt = 1; - else - mi_switch(SW_INVOL, NULL); - -#else /* PREEMPTION */ - running_thread->td_flags |= TDF_NEEDRESCHED; -#endif /* PREEMPTION */ - return; -} - -#else /* SMP */ -{ - struct thread *running_thread; - int worst_pri; - struct ksegrp *kg; - cpumask_t cpumask,dontuse; - struct pcpu *pc; - struct pcpu *best_pcpu; - struct thread *cputhread; - - mtx_assert(&sched_lock, MA_OWNED); - - running_thread = curthread; - -#if !defined(KSEG_PEEMPT_BEST_CPU) - if (running_thread->td_ksegrp != td->td_ksegrp) { -#endif - kg = td->td_ksegrp; - - /* if someone is ahead of this thread, wait our turn */ - if (td != TAILQ_FIRST(&kg->kg_runq)) - return; - - worst_pri = td->td_priority; - best_pcpu = NULL; - dontuse = stopped_cpus | idle_cpus_mask; - - /* - * Find a cpu with the worst priority that runs at thread from - * the same ksegrp - if multiple exist give first the last run - * cpu and then the current cpu priority - */ - - SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { - cpumask = pc->pc_cpumask; - cputhread = pc->pc_curthread; - - if ((cpumask & dontuse) || - cputhread->td_ksegrp != kg) - continue; - - if (cputhread->td_priority > worst_pri) { - worst_pri = cputhread->td_priority; - best_pcpu = pc; - continue; - } - - if (cputhread->td_priority == worst_pri && - best_pcpu != NULL && - (td->td_lastcpu == pc->pc_cpuid || - (PCPU_GET(cpumask) == cpumask && - td->td_lastcpu != best_pcpu->pc_cpuid))) - best_pcpu = pc; - } - - /* Check if we need to preempt someone */ - if (best_pcpu == NULL) - return; - -#if defined(IPI_PREEMPTION) && defined(PREEMPTION) -#if !defined(FULL_PREEMPTION) - if (td->td_priority <= PRI_MAX_ITHD) -#endif /* ! FULL_PREEMPTION */ - { - ipi_selected(best_pcpu->pc_cpumask, IPI_PREEMPT); - return; - } -#endif /* defined(IPI_PREEMPTION) && defined(PREEMPTION) */ - - if (PCPU_GET(cpuid) != best_pcpu->pc_cpuid) { - best_pcpu->pc_curthread->td_flags |= TDF_NEEDRESCHED; - ipi_selected(best_pcpu->pc_cpumask, IPI_AST); - return; - } -#if !defined(KSEG_PEEMPT_BEST_CPU) - } -#endif - - if (td->td_priority >= running_thread->td_priority) - return; -#ifdef PREEMPTION - -#if !defined(FULL_PREEMPTION) - if (td->td_priority > PRI_MAX_ITHD) { - running_thread->td_flags |= TDF_NEEDRESCHED; - } -#endif /* ! FULL_PREEMPTION */ - - if (running_thread->td_critnest > 1) - running_thread->td_owepreempt = 1; - else - mi_switch(SW_INVOL, NULL); - -#else /* PREEMPTION */ - running_thread->td_flags |= TDF_NEEDRESCHED; -#endif /* PREEMPTION */ - return; -} -#endif /* !SMP */ - - -int limitcount; -#endif void setrunqueue(struct thread *td, int flags) { -#ifdef KSE - struct ksegrp *kg; - struct thread *td2; - struct thread *tda; - CTR3(KTR_RUNQ, "setrunqueue: td:%p kg:%p pid:%d", - td, td->td_ksegrp, td->td_proc->p_pid); -#else CTR2(KTR_RUNQ, "setrunqueue: td:%p pid:%d", td, td->td_proc->p_pid); -#endif CTR5(KTR_SCHED, "setrunqueue: %p(%s) prio %d by %p(%s)", td, td->td_proc->p_comm, td->td_priority, curthread, curthread->td_proc->p_comm); @@ -557,101 +190,7 @@ setrunqueue(struct thread *td, int flags) KASSERT((TD_CAN_RUN(td) || TD_IS_RUNNING(td)), ("setrunqueue: bad thread state")); TD_SET_RUNQ(td); -#ifdef KSE - kg = td->td_ksegrp; - if ((td->td_proc->p_flag & P_HADTHREADS) == 0) { - /* - * Common path optimisation: Only one of everything - * and the KSE is always already attached. - * Totally ignore the ksegrp run queue. - */ - if (kg->kg_avail_opennings != 1) { - if (limitcount < 1) { - limitcount++; - printf("pid %d: corrected slot count (%d->1)\n", - td->td_proc->p_pid, kg->kg_avail_opennings); - - } - kg->kg_avail_opennings = 1; - } - sched_add(td, flags); - return; - } - - /* - * If the concurrency has reduced, and we would go in the - * assigned section, then keep removing entries from the - * system run queue, until we are not in that section - * or there is room for us to be put in that section. - * What we MUST avoid is the case where there are threads of less - * priority than the new one scheduled, but it can not - * be scheduled itself. That would lead to a non contiguous set - * of scheduled threads, and everything would break. - */ - tda = kg->kg_last_assigned; - while ((kg->kg_avail_opennings <= 0) && - (tda && (tda->td_priority > td->td_priority))) { - /* - * None free, but there is one we can commandeer. - */ - CTR2(KTR_RUNQ, - "setrunqueue: kg:%p: take slot from td: %p", kg, tda); - sched_rem(tda); - tda = kg->kg_last_assigned = - TAILQ_PREV(tda, threadqueue, td_runq); - } - - /* - * Add the thread to the ksegrp's run queue at - * the appropriate place. - */ - TAILQ_FOREACH(td2, &kg->kg_runq, td_runq) { - if (td2->td_priority > td->td_priority) { - TAILQ_INSERT_BEFORE(td2, td, td_runq); - break; - } - } - if (td2 == NULL) { - /* We ran off the end of the TAILQ or it was empty. */ - TAILQ_INSERT_TAIL(&kg->kg_runq, td, td_runq); - } - - /* - * If we have a slot to use, then put the thread on the system - * run queue and if needed, readjust the last_assigned pointer. - * it may be that we need to schedule something anyhow - * even if the availabel slots are -ve so that - * all the items < last_assigned are scheduled. - */ - if (kg->kg_avail_opennings > 0) { - if (tda == NULL) { - /* - * No pre-existing last assigned so whoever is first - * gets the slot.. (maybe us) - */ - td2 = TAILQ_FIRST(&kg->kg_runq); - kg->kg_last_assigned = td2; - } else if (tda->td_priority > td->td_priority) { - td2 = td; - } else { - /* - * We are past last_assigned, so - * give the next slot to whatever is next, - * which may or may not be us. - */ - td2 = TAILQ_NEXT(tda, td_runq); - kg->kg_last_assigned = td2; - } - sched_add(td2, flags); - } else { - CTR3(KTR_RUNQ, "setrunqueue: held: td%p kg%p pid%d", - td, td->td_ksegrp, td->td_proc->p_pid); - if ((flags & SRQ_YIELDING) == 0) - maybe_preempt_in_ksegrp(td); - } -#else sched_add(td, flags); -#endif } /* @@ -737,14 +276,14 @@ maybe_preempt(struct thread *td) * to the new thread. */ ctd = curthread; - KASSERT ((ctd->td_kse != NULL && ctd->td_kse->ke_thread == ctd), + KASSERT ((ctd->td_sched != NULL && ctd->td_sched->ts_thread == ctd), ("thread has no (or wrong) sched-private part.")); KASSERT((td->td_inhibitors == 0), ("maybe_preempt: trying to run inhibitted thread")); pri = td->td_priority; cpri = ctd->td_priority; if (panicstr != NULL || pri >= cpri || cold /* || dumping */ || - TD_IS_INHIBITED(ctd) || td->td_kse->ke_state != KES_THREAD) + TD_IS_INHIBITED(ctd) || td->td_sched->ts_state != TSS_THREAD) return (0); #ifndef FULL_PREEMPTION if (pri > PRI_MAX_ITHD && cpri < PRI_MIN_IDLE) @@ -762,25 +301,7 @@ maybe_preempt(struct thread *td) * Thread is runnable but not yet put on system run queue. */ MPASS(TD_ON_RUNQ(td)); - MPASS(td->td_sched->ke_state != KES_ONRUNQ); -#ifdef KSE - if (td->td_proc->p_flag & P_HADTHREADS) { - /* - * If this is a threaded process we actually ARE on the - * ksegrp run queue so take it off that first. - * Also undo any damage done to the last_assigned pointer. - * XXX Fix setrunqueue so this isn't needed - */ - struct ksegrp *kg; - - kg = td->td_ksegrp; - if (kg->kg_last_assigned == td) - kg->kg_last_assigned = - TAILQ_PREV(td, threadqueue, td_runq); - TAILQ_REMOVE(&kg->kg_runq, td, td_runq); - } - -#endif + MPASS(td->td_sched->ts_state != TSS_ONRUNQ); TD_SET_RUNNING(td); CTR3(KTR_PROC, "preempting to thread %p (pid %d, %s)\n", td, td->td_proc->p_pid, td->td_proc->p_comm); @@ -880,25 +401,25 @@ runq_setbit(struct runq *rq, int pri) } /* - * Add the KSE to the queue specified by its priority, and set the + * Add the thread to the queue specified by its priority, and set the * corresponding status bit. */ void -runq_add(struct runq *rq, struct kse *ke, int flags) +runq_add(struct runq *rq, struct td_sched *ts, int flags) { struct rqhead *rqh; int pri; - pri = ke->ke_thread->td_priority / RQ_PPQ; - ke->ke_rqindex = pri; + pri = ts->ts_thread->td_priority / RQ_PPQ; + ts->ts_rqindex = pri; runq_setbit(rq, pri); rqh = &rq->rq_queues[pri]; - CTR5(KTR_RUNQ, "runq_add: td=%p ke=%p pri=%d %d rqh=%p", - ke->ke_thread, ke, ke->ke_thread->td_priority, pri, rqh); + CTR5(KTR_RUNQ, "runq_add: td=%p ts=%p pri=%d %d rqh=%p", + ts->ts_thread, ts, ts->ts_thread->td_priority, pri, rqh); if (flags & SRQ_PREEMPTED) { - TAILQ_INSERT_HEAD(rqh, ke, ke_procq); + TAILQ_INSERT_HEAD(rqh, ts, ts_procq); } else { - TAILQ_INSERT_TAIL(rqh, ke, ke_procq); + TAILQ_INSERT_TAIL(rqh, ts, ts_procq); } } @@ -933,11 +454,11 @@ SYSCTL_INT(_kern_sched, OID_AUTO, runq_fuzz, CTLFLAG_RW, &runq_fuzz, 0, ""); /* * Find the highest priority process on the run queue. */ -struct kse * +struct td_sched * runq_choose(struct runq *rq) { struct rqhead *rqh; - struct kse *ke; + struct td_sched *ts; int pri; mtx_assert(&sched_lock, MA_OWNED); @@ -952,23 +473,23 @@ runq_choose(struct runq *rq) */ int count = runq_fuzz; int cpu = PCPU_GET(cpuid); - struct kse *ke2; - ke2 = ke = TAILQ_FIRST(rqh); + struct td_sched *ts2; + ts2 = ts = TAILQ_FIRST(rqh); - while (count-- && ke2) { - if (ke->ke_thread->td_lastcpu == cpu) { - ke = ke2; + while (count-- && ts2) { + if (ts->ts_thread->td_lastcpu == cpu) { + ts = ts2; break; } - ke2 = TAILQ_NEXT(ke2, ke_procq); + ts2 = TAILQ_NEXT(ts2, ts_procq); } } else #endif - ke = TAILQ_FIRST(rqh); - KASSERT(ke != NULL, ("runq_choose: no proc on busy queue")); + ts = TAILQ_FIRST(rqh); + KASSERT(ts != NULL, ("runq_choose: no proc on busy queue")); CTR3(KTR_RUNQ, - "runq_choose: pri=%d kse=%p rqh=%p", pri, ke, rqh); - return (ke); + "runq_choose: pri=%d td_sched=%p rqh=%p", pri, ts, rqh); + return (ts); } CTR1(KTR_RUNQ, "runq_choose: idleproc pri=%d", pri); @@ -976,28 +497,24 @@ runq_choose(struct runq *rq) } /* - * Remove the KSE from the queue specified by its priority, and clear the + * Remove the thread from the queue specified by its priority, and clear the * corresponding status bit if the queue becomes empty. - * Caller must set ke->ke_state afterwards. + * Caller must set ts->ts_state afterwards. */ void -runq_remove(struct runq *rq, struct kse *ke) +runq_remove(struct runq *rq, struct td_sched *ts) { struct rqhead *rqh; int pri; -#ifdef KSE - KASSERT(ke->ke_proc->p_sflag & PS_INMEM, -#else - KASSERT(ke->ke_thread->td_proc->p_sflag & PS_INMEM, -#endif + KASSERT(ts->ts_thread->td_proc->p_sflag & PS_INMEM, ("runq_remove: process swapped out")); - pri = ke->ke_rqindex; + pri = ts->ts_rqindex; rqh = &rq->rq_queues[pri]; - CTR5(KTR_RUNQ, "runq_remove: td=%p, ke=%p pri=%d %d rqh=%p", - ke->ke_thread, ke, ke->ke_thread->td_priority, pri, rqh); - KASSERT(ke != NULL, ("runq_remove: no proc on busy queue")); - TAILQ_REMOVE(rqh, ke, ke_procq); + CTR5(KTR_RUNQ, "runq_remove: td=%p, ts=%p pri=%d %d rqh=%p", + ts->ts_thread, ts, ts->ts_thread->td_priority, pri, rqh); + KASSERT(ts != NULL, ("runq_remove: no proc on busy queue")); + TAILQ_REMOVE(rqh, ts, ts_procq); if (TAILQ_EMPTY(rqh)) { CTR0(KTR_RUNQ, "runq_remove: empty"); runq_clrbit(rq, pri); @@ -1008,23 +525,17 @@ runq_remove(struct runq *rq, struct kse *ke) #include extern struct mtx kse_zombie_lock; -#ifdef KSE /* * Allocate scheduler specific per-process resources. - * The thread and ksegrp have already been linked in. - * In this case just set the default concurrency value. + * The thread and proc have already been linked in. * * Called from: * proc_init() (UMA init method) */ void -sched_newproc(struct proc *p, struct ksegrp *kg, struct thread *td) +sched_newproc(struct proc *p, struct thread *td) { - - /* This can go in sched_fork */ - sched_init_concurrency(kg); } -#endif /* * thread is being either created or recycled. @@ -1037,37 +548,27 @@ sched_newproc(struct proc *p, struct ksegrp *kg, struct thread *td) void sched_newthread(struct thread *td) { - struct td_sched *ke; + struct td_sched *ts; - ke = (struct td_sched *) (td + 1); - bzero(ke, sizeof(*ke)); - td->td_sched = ke; - ke->ke_thread = td; - ke->ke_state = KES_THREAD; + ts = (struct td_sched *) (td + 1); + bzero(ts, sizeof(*ts)); + td->td_sched = ts; + ts->ts_thread = td; + ts->ts_state = TSS_THREAD; } -#ifdef KSE /* - * Set up an initial concurrency of 1 - * and set the given thread (if given) to be using that - * concurrency slot. - * May be used "offline"..before the ksegrp is attached to the world - * and thus wouldn't need schedlock in that case. * Called from: * thr_create() * proc_init() (UMA) via sched_newproc() */ void -sched_init_concurrency(struct ksegrp *kg) +sched_init_concurrency(struct proc *p) { - - CTR1(KTR_RUNQ,"kg %p init slots and concurrency to 1", kg); - kg->kg_concurrency = 1; - kg->kg_avail_opennings = 1; } /* - * Change the concurrency of an existing ksegrp to N + * Change the concurrency of an existing proc to N * Called from: * kse_create() * kse_exit() @@ -1075,16 +576,8 @@ sched_init_concurrency(struct ksegrp *kg) * thread_single() */ void -sched_set_concurrency(struct ksegrp *kg, int concurrency) +sched_set_concurrency(struct proc *p, int concurrency) { - - CTR4(KTR_RUNQ,"kg %p set concurrency to %d, slots %d -> %d", - kg, - concurrency, - kg->kg_avail_opennings, - kg->kg_avail_opennings + (concurrency - kg->kg_concurrency)); - kg->kg_avail_opennings += (concurrency - kg->kg_concurrency); - kg->kg_concurrency = concurrency; } /* @@ -1099,10 +592,6 @@ sched_set_concurrency(struct ksegrp *kg, int concurrency) void sched_thread_exit(struct thread *td) { - - SLOT_RELEASE(td->td_ksegrp); - slot_fill(td->td_ksegrp); } -#endif #endif /* KERN_SWITCH_INCLUDE */ diff --git a/sys/kern/kern_thr.c b/sys/kern/kern_thr.c index 37e3df2cdd9a..2769f458c8af 100644 --- a/sys/kern/kern_thr.c +++ b/sys/kern/kern_thr.c @@ -142,18 +142,12 @@ create_thread(struct thread *td, mcontext_t *ctx, { stack_t stack; struct thread *newtd; -#ifdef KSE - struct ksegrp *kg, *newkg; -#endif struct proc *p; long id; int error; error = 0; p = td->td_proc; -#ifdef KSE - kg = td->td_ksegrp; -#endif /* Have race condition but it is cheap. */ if (p->p_numthreads >= max_threads_per_proc) @@ -177,7 +171,7 @@ create_thread(struct thread *td, mcontext_t *ctx, } } - /* Initialize our td and new ksegrp.. */ + /* Initialize our td */ newtd = thread_alloc(); /* @@ -229,42 +223,15 @@ create_thread(struct thread *td, mcontext_t *ctx, } } -#ifdef KSE - newkg = ksegrp_alloc(); - bzero(&newkg->kg_startzero, - __rangeof(struct ksegrp, kg_startzero, kg_endzero)); - bcopy(&kg->kg_startcopy, &newkg->kg_startcopy, - __rangeof(struct ksegrp, kg_startcopy, kg_endcopy)); - sched_init_concurrency(newkg); PROC_LOCK(td->td_proc); - td->td_proc->p_flag |= P_HADTHREADS; - newtd->td_sigmask = td->td_sigmask; - mtx_lock_spin(&sched_lock); - ksegrp_link(newkg, p); - thread_link(newtd, newkg); - PROC_UNLOCK(p); -#else - PROC_LOCK(td->td_proc); td->td_proc->p_flag |= P_HADTHREADS; newtd->td_sigmask = td->td_sigmask; mtx_lock_spin(&sched_lock); thread_link(newtd, p); PROC_UNLOCK(p); -#endif -#ifdef KSE /* let the scheduler know about these things. */ - sched_fork_ksegrp(td, newkg); sched_fork_thread(td, newtd); - if (rtp != NULL) { - if (!(kg->kg_pri_class == PRI_TIMESHARE && - rtp->type == RTP_PRIO_NORMAL)) { - rtp_to_pri(rtp, newkg); - sched_prio(newtd, newkg->kg_user_pri); - } /* ignore timesharing class */ - } -#else - sched_fork(td, newtd); if (rtp != NULL) { if (!(td->td_pri_class == PRI_TIMESHARE && rtp->type == RTP_PRIO_NORMAL)) { @@ -272,7 +239,6 @@ create_thread(struct thread *td, mcontext_t *ctx, sched_prio(newtd, newtd->td_user_pri); } /* ignore timesharing class */ } -#endif TD_SET_CAN_RUN(newtd); /* if ((flags & THR_SUSPENDED) == 0) */ setrunqueue(newtd, SRQ_BORING); diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c index 74af901df50a..1f9e80fb240c 100644 --- a/sys/kern/kern_thread.c +++ b/sys/kern/kern_thread.c @@ -50,16 +50,9 @@ __FBSDID("$FreeBSD$"); #include #include -#ifdef KSE -/* - * KSEGRP related storage. - */ -static uma_zone_t ksegrp_zone; -#else /* * thread related storage. */ -#endif static uma_zone_t thread_zone; /* DEBUG ONLY */ @@ -85,9 +78,6 @@ int virtual_cpu; #endif TAILQ_HEAD(, thread) zombie_threads = TAILQ_HEAD_INITIALIZER(zombie_threads); -#ifdef KSE -TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps); -#endif struct mtx kse_zombie_lock; MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN); @@ -228,59 +218,6 @@ thread_fini(void *mem, int size) vm_thread_dispose(td); } -#ifdef KSE -/* - * Initialize type-stable parts of a ksegrp (when newly created). - */ -static int -ksegrp_ctor(void *mem, int size, void *arg, int flags) -{ - struct ksegrp *kg; - - kg = (struct ksegrp *)mem; - bzero(mem, size); - kg->kg_sched = (struct kg_sched *)&kg[1]; - return (0); -} - -void -ksegrp_link(struct ksegrp *kg, struct proc *p) -{ - - TAILQ_INIT(&kg->kg_threads); - TAILQ_INIT(&kg->kg_runq); /* links with td_runq */ - TAILQ_INIT(&kg->kg_upcalls); /* all upcall structure in ksegrp */ - kg->kg_proc = p; - /* - * the following counters are in the -zero- section - * and may not need clearing - */ - kg->kg_numthreads = 0; - kg->kg_numupcalls = 0; - /* link it in now that it's consistent */ - p->p_numksegrps++; - TAILQ_INSERT_HEAD(&p->p_ksegrps, kg, kg_ksegrp); -} - -/* - * Called from: - * thread-exit() - */ -void -ksegrp_unlink(struct ksegrp *kg) -{ - struct proc *p; - - mtx_assert(&sched_lock, MA_OWNED); - KASSERT((kg->kg_numthreads == 0), ("ksegrp_unlink: residual threads")); - KASSERT((kg->kg_numupcalls == 0), ("ksegrp_unlink: residual upcalls")); - - p = kg->kg_proc; - TAILQ_REMOVE(&p->p_ksegrps, kg, kg_ksegrp); - p->p_numksegrps--; -} -#endif - /* * For a newly created process, * link up all the structures and its initial threads etc. @@ -290,18 +227,10 @@ ksegrp_unlink(struct ksegrp *kg) * proc_init() */ void -#ifdef KSE -proc_linkup(struct proc *p, struct ksegrp *kg, struct thread *td) -#else proc_linkup(struct proc *p, struct thread *td) -#endif { - -#ifdef KSE - TAILQ_INIT(&p->p_ksegrps); /* all ksegrps in proc */ -#endif TAILQ_INIT(&p->p_threads); /* all threads in proc */ - TAILQ_INIT(&p->p_suspended); /* Threads suspended */ + TAILQ_INIT(&p->p_upcalls); /* upcall list */ sigqueue_init(&p->p_sigqueue, p); p->p_ksi = ksiginfo_alloc(1); if (p->p_ksi != NULL) { @@ -309,17 +238,8 @@ proc_linkup(struct proc *p, struct thread *td) p->p_ksi->ksi_flags = KSI_EXT | KSI_INS; } LIST_INIT(&p->p_mqnotifier); -#ifdef KSE - p->p_numksegrps = 0; -#endif p->p_numthreads = 0; - -#ifdef KSE - ksegrp_link(kg, p); - thread_link(td, kg); -#else thread_link(td, p); -#endif } /* @@ -336,37 +256,22 @@ threadinit(void) thread_ctor, thread_dtor, thread_init, thread_fini, UMA_ALIGN_CACHE, 0); #ifdef KSE - ksegrp_zone = uma_zcreate("KSEGRP", sched_sizeof_ksegrp(), - ksegrp_ctor, NULL, NULL, NULL, - UMA_ALIGN_CACHE, 0); kseinit(); /* set up kse specific stuff e.g. upcall zone*/ #endif } /* * Stash an embarasingly extra thread into the zombie thread queue. + * Use the slpq as that must be unused by now. */ void thread_stash(struct thread *td) { mtx_lock_spin(&kse_zombie_lock); - TAILQ_INSERT_HEAD(&zombie_threads, td, td_runq); + TAILQ_INSERT_HEAD(&zombie_threads, td, td_slpq); mtx_unlock_spin(&kse_zombie_lock); } -#ifdef KSE -/* - * Stash an embarasingly extra ksegrp into the zombie ksegrp queue. - */ -void -ksegrp_stash(struct ksegrp *kg) -{ - mtx_lock_spin(&kse_zombie_lock); - TAILQ_INSERT_HEAD(&zombie_ksegrps, kg, kg_ksegrp); - mtx_unlock_spin(&kse_zombie_lock); -} -#endif - /* * Reap zombie kse resource. */ @@ -374,65 +279,27 @@ void thread_reap(void) { struct thread *td_first, *td_next; -#ifdef KSE - struct ksegrp *kg_first, * kg_next; -#endif /* * Don't even bother to lock if none at this instant, * we really don't care about the next instant.. */ -#ifdef KSE - if ((!TAILQ_EMPTY(&zombie_threads)) - || (!TAILQ_EMPTY(&zombie_ksegrps))) { -#else if (!TAILQ_EMPTY(&zombie_threads)) { -#endif mtx_lock_spin(&kse_zombie_lock); td_first = TAILQ_FIRST(&zombie_threads); -#ifdef KSE - kg_first = TAILQ_FIRST(&zombie_ksegrps); -#endif if (td_first) TAILQ_INIT(&zombie_threads); -#ifdef KSE - if (kg_first) - TAILQ_INIT(&zombie_ksegrps); -#endif mtx_unlock_spin(&kse_zombie_lock); while (td_first) { - td_next = TAILQ_NEXT(td_first, td_runq); + td_next = TAILQ_NEXT(td_first, td_slpq); if (td_first->td_ucred) crfree(td_first->td_ucred); thread_free(td_first); td_first = td_next; } -#ifdef KSE - while (kg_first) { - kg_next = TAILQ_NEXT(kg_first, kg_ksegrp); - ksegrp_free(kg_first); - kg_first = kg_next; - } - /* - * there will always be a thread on the list if one of these - * is there. - */ - kse_GC(); -#endif } } -#ifdef KSE -/* - * Allocate a ksegrp. - */ -struct ksegrp * -ksegrp_alloc(void) -{ - return (uma_zalloc(ksegrp_zone, M_WAITOK)); -} -#endif - /* * Allocate a thread. */ @@ -444,16 +311,6 @@ thread_alloc(void) return (uma_zalloc(thread_zone, M_WAITOK)); } -#ifdef KSE -/* - * Deallocate a ksegrp. - */ -void -ksegrp_free(struct ksegrp *td) -{ - uma_zfree(ksegrp_zone, td); -} -#endif /* * Deallocate a thread. @@ -503,23 +360,14 @@ thread_exit(void) uint64_t new_switchtime; struct thread *td; struct proc *p; -#ifdef KSE - struct ksegrp *kg; -#endif td = curthread; -#ifdef KSE - kg = td->td_ksegrp; -#endif p = td->td_proc; mtx_assert(&sched_lock, MA_OWNED); mtx_assert(&Giant, MA_NOTOWNED); PROC_LOCK_ASSERT(p, MA_OWNED); KASSERT(p != NULL, ("thread exiting without a process")); -#ifdef KSE - KASSERT(kg != NULL, ("thread exiting without a kse group")); -#endif CTR3(KTR_PROC, "thread_exit: thread %p (pid %ld, %s)", td, (long)p->p_pid, p->p_comm); KASSERT(TAILQ_EMPTY(&td->td_sigqueue.sq_list), ("signal pending")); @@ -583,13 +431,8 @@ thread_exit(void) if (p->p_flag & P_HADTHREADS) { if (p->p_numthreads > 1) { thread_unlink(td); -#ifdef KSE - /* XXX first arg not used in 4BSD or ULE */ sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); -#else - sched_exit(p, td); -#endif /* * The test below is NOT true if we are the @@ -614,38 +457,9 @@ thread_exit(void) * there somehow. */ upcall_remove(td); +#endif - /* - * If the thread we unlinked above was the last one, - * then this ksegrp should go away too. - */ - if (kg->kg_numthreads == 0) { - /* - * let the scheduler know about this in case - * it needs to recover stats or resources. - * Theoretically we could let - * sched_exit_ksegrp() do the equivalent of - * setting the concurrency to 0 - * but don't do it yet to avoid changing - * the existing scheduler code until we - * are ready. - * We supply a random other ksegrp - * as the recipient of any built up - * cpu usage etc. (If the scheduler wants it). - * XXXKSE - * This is probably not fair so think of - * a better answer. - */ - sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td); - sched_set_concurrency(kg, 0); /* XXX TEMP */ - ksegrp_unlink(kg); - ksegrp_stash(kg); - } -#endif PROC_UNLOCK(p); -#ifdef KSE - td->td_ksegrp = NULL; -#endif PCPU_SET(deadthread, td); } else { /* @@ -689,9 +503,6 @@ thread_wait(struct proc *p) mtx_assert(&Giant, MA_NOTOWNED); KASSERT((p->p_numthreads == 1), ("Multiple threads in wait1()")); -#ifdef KSE - KASSERT((p->p_numksegrps == 1), ("Multiple ksegrps in wait1()")); -#endif FOREACH_THREAD_IN_PROC(p, td) { #ifdef KSE if (td->td_standin != NULL) { @@ -718,46 +529,22 @@ thread_wait(struct proc *p) * The thread is linked as if running but no KSE assigned. * Called from: * proc_linkup() - * ifdef KSE * thread_schedule_upcall() - * endif * thr_create() */ void -#ifdef KSE -thread_link(struct thread *td, struct ksegrp *kg) -#else thread_link(struct thread *td, struct proc *p) -#endif { -#ifdef KSE - struct proc *p; -#endif -#ifdef KSE - p = kg->kg_proc; -#endif td->td_state = TDS_INACTIVE; td->td_proc = p; -#ifdef KSE - td->td_ksegrp = kg; -#endif td->td_flags = 0; -#ifdef KSE - td->td_kflags = 0; -#endif LIST_INIT(&td->td_contested); sigqueue_init(&td->td_sigqueue, p); callout_init(&td->td_slpcallout, CALLOUT_MPSAFE); TAILQ_INSERT_HEAD(&p->p_threads, td, td_plist); -#ifdef KSE - TAILQ_INSERT_HEAD(&kg->kg_threads, td, td_kglist); -#endif p->p_numthreads++; -#ifdef KSE - kg->kg_numthreads++; -#endif } /* @@ -781,7 +568,7 @@ thread_unthread(struct thread *td) thread_stash(td->td_standin); td->td_standin = NULL; } - sched_set_concurrency(td->td_ksegrp, 1); + sched_set_concurrency(p, 1); #else p->p_flag &= ~P_HADTHREADS; #endif @@ -795,23 +582,12 @@ void thread_unlink(struct thread *td) { struct proc *p = td->td_proc; -#ifdef KSE - struct ksegrp *kg = td->td_ksegrp; -#endif mtx_assert(&sched_lock, MA_OWNED); TAILQ_REMOVE(&p->p_threads, td, td_plist); p->p_numthreads--; -#ifdef KSE - TAILQ_REMOVE(&kg->kg_threads, td, td_kglist); - kg->kg_numthreads--; -#endif /* could clear a few other things here */ -#ifdef KSE - /* Must NOT clear links to proc and ksegrp! */ -#else /* Must NOT clear links to proc! */ -#endif } /* @@ -1040,8 +816,7 @@ thread_suspend_check(int return_instead) /* * When a thread suspends, it just - * moves to the processes's suspend queue - * and stays there. + * gets taken off all queues. */ thread_suspend_one(td); if (return_instead == 0) { @@ -1074,7 +849,6 @@ thread_suspend_one(struct thread *td) KASSERT(!TD_IS_SUSPENDED(td), ("already suspended")); p->p_suspcount++; TD_SET_SUSPENDED(td); - TAILQ_INSERT_TAIL(&p->p_suspended, td, td_runq); } void @@ -1084,7 +858,7 @@ thread_unsuspend_one(struct thread *td) mtx_assert(&sched_lock, MA_OWNED); PROC_LOCK_ASSERT(p, MA_OWNED); - TAILQ_REMOVE(&p->p_suspended, td, td_runq); + KASSERT(TD_IS_SUSPENDED(td), ("Thread not suspended")); TD_CLR_SUSPENDED(td); p->p_suspcount--; setrunnable(td); @@ -1101,8 +875,10 @@ thread_unsuspend(struct proc *p) mtx_assert(&sched_lock, MA_OWNED); PROC_LOCK_ASSERT(p, MA_OWNED); if (!P_SHOULDSTOP(p)) { - while ((td = TAILQ_FIRST(&p->p_suspended))) { - thread_unsuspend_one(td); + FOREACH_THREAD_IN_PROC(p, td) { + if (TD_IS_SUSPENDED(td)) { + thread_unsuspend_one(td); + } } } else if ((P_SHOULDSTOP(p) == P_STOPPED_SINGLE) && (p->p_numthreads == p->p_suspcount)) { @@ -1137,8 +913,10 @@ thread_single_end(void) * to continue however as this is a bad place to stop. */ if ((p->p_numthreads != 1) && (!P_SHOULDSTOP(p))) { - while ((td = TAILQ_FIRST(&p->p_suspended))) { - thread_unsuspend_one(td); + FOREACH_THREAD_IN_PROC(p, td) { + if (TD_IS_SUSPENDED(td)) { + thread_unsuspend_one(td); + } } } mtx_unlock_spin(&sched_lock); diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c index 328fb576626c..645f02df8643 100644 --- a/sys/kern/kern_umtx.c +++ b/sys/kern/kern_umtx.c @@ -167,15 +167,9 @@ struct umtxq_chain { * if it is using 100%CPU, this is unfair to other processes. */ -#ifdef KSE -#define UPRI(td) (((td)->td_ksegrp->kg_user_pri >= PRI_MIN_TIMESHARE &&\ - (td)->td_ksegrp->kg_user_pri <= PRI_MAX_TIMESHARE) ?\ - PRI_MAX_TIMESHARE : (td)->td_ksegrp->kg_user_pri) -#else #define UPRI(td) (((td)->td_user_pri >= PRI_MIN_TIMESHARE &&\ (td)->td_user_pri <= PRI_MAX_TIMESHARE) ?\ PRI_MAX_TIMESHARE : (td)->td_user_pri) -#endif #define GOLDEN_RATIO_PRIME 2654404609U #define UMTX_CHAINS 128 diff --git a/sys/kern/ksched.c b/sys/kern/ksched.c index 4127367cb442..785c04c38c2e 100644 --- a/sys/kern/ksched.c +++ b/sys/kern/ksched.c @@ -105,11 +105,7 @@ getscheduler(struct ksched *ksched, struct thread *td, int *policy) int e = 0; mtx_lock_spin(&sched_lock); -#ifdef KSE - pri_to_rtp(td->td_ksegrp, &rtp); -#else pri_to_rtp(td, &rtp); -#endif mtx_unlock_spin(&sched_lock); switch (rtp.type) { @@ -156,11 +152,7 @@ ksched_getparam(struct ksched *ksched, struct rtprio rtp; mtx_lock_spin(&sched_lock); -#ifdef KSE - pri_to_rtp(td->td_ksegrp, &rtp); -#else pri_to_rtp(td, &rtp); -#endif mtx_unlock_spin(&sched_lock); if (RTP_PRIO_IS_REALTIME(rtp.type)) param->sched_priority = rtpprio_to_p4prio(rtp.prio); @@ -181,9 +173,6 @@ ksched_setscheduler(struct ksched *ksched, { int e = 0; struct rtprio rtp; -#ifdef KSE - struct ksegrp *kg = td->td_ksegrp; -#endif switch(policy) { @@ -198,20 +187,7 @@ ksched_setscheduler(struct ksched *ksched, ? RTP_PRIO_FIFO : RTP_PRIO_REALTIME; mtx_lock_spin(&sched_lock); -#ifdef KSE - rtp_to_pri(&rtp, kg); - FOREACH_THREAD_IN_GROUP(kg, td) { /* XXXKSE */ - if (TD_IS_RUNNING(td)) { - td->td_flags |= TDF_NEEDRESCHED; - } else if (TD_ON_RUNQ(td)) { - if (td->td_priority > kg->kg_user_pri) { - sched_prio(td, kg->kg_user_pri); - } - } - } -#else rtp_to_pri(&rtp, td); -#endif mtx_unlock_spin(&sched_lock); } else @@ -225,28 +201,7 @@ ksched_setscheduler(struct ksched *ksched, rtp.type = RTP_PRIO_NORMAL; rtp.prio = p4prio_to_rtpprio(param->sched_priority); mtx_lock_spin(&sched_lock); -#ifdef KSE - rtp_to_pri(&rtp, kg); - - /* XXX Simply revert to whatever we had for last - * normal scheduler priorities. - * This puts a requirement - * on the scheduling code: You must leave the - * scheduling info alone. - */ - FOREACH_THREAD_IN_GROUP(kg, td) { - if (TD_IS_RUNNING(td)) { - td->td_flags |= TDF_NEEDRESCHED; - } else if (TD_ON_RUNQ(td)) { - if (td->td_priority > kg->kg_user_pri) { - sched_prio(td, kg->kg_user_pri); - } - } - - } -#else rtp_to_pri(&rtp, td); -#endif mtx_unlock_spin(&sched_lock); } break; diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c index dc9d288c475e..3fe26825a97a 100644 --- a/sys/kern/sched_4bsd.c +++ b/sys/kern/sched_4bsd.c @@ -37,8 +37,6 @@ __FBSDID("$FreeBSD$"); #include "opt_hwpmc_hooks.h" -#define kse td_sched - #include #include #include @@ -75,99 +73,38 @@ __FBSDID("$FreeBSD$"); #endif #define NICE_WEIGHT 1 /* Priorities per nice level. */ -#ifdef KSE -/* - * The schedulable entity that can be given a context to run. - * A process may have several of these. Probably one per processor - * but possibly a few more. In this universe they are grouped - * with a KSEG that contains the priority and niceness - * for the group. - */ -#else /* * The schedulable entity that runs a context. - * A process may have several of these. Probably one per processor - * but posibly a few more. + * This is an extension to the thread structure and is tailored to + * the requirements of this scheduler */ -#endif -struct kse { - TAILQ_ENTRY(kse) ke_procq; /* (j/z) Run queue. */ - struct thread *ke_thread; /* (*) Active associated thread. */ - fixpt_t ke_pctcpu; /* (j) %cpu during p_swtime. */ - u_char ke_rqindex; /* (j) Run queue index. */ +struct td_sched { + TAILQ_ENTRY(td_sched) ts_procq; /* (j/z) Run queue. */ + struct thread *ts_thread; /* (*) Active associated thread. */ + fixpt_t ts_pctcpu; /* (j) %cpu during p_swtime. */ + u_char ts_rqindex; /* (j) Run queue index. */ enum { - KES_THREAD = 0x0, /* slaved to thread state */ - KES_ONRUNQ - } ke_state; /* (j) KSE status. */ - int ke_cpticks; /* (j) Ticks of cpu time. */ - struct runq *ke_runq; /* runq the kse is currently on */ + TSS_THREAD = 0x0, /* slaved to thread state */ + TSS_ONRUNQ + } ts_state; /* (j) TD_STAT in scheduler status. */ + int ts_cpticks; /* (j) Ticks of cpu time. */ + struct runq *ts_runq; /* runq the thread is currently on */ }; -#ifdef KSE -#define ke_proc ke_thread->td_proc -#define ke_ksegrp ke_thread->td_ksegrp -#endif - -#define td_kse td_sched - /* flags kept in td_flags */ -#define TDF_DIDRUN TDF_SCHED0 /* KSE actually ran. */ -#define TDF_EXIT TDF_SCHED1 /* KSE is being killed. */ +#define TDF_DIDRUN TDF_SCHED0 /* thread actually ran. */ +#define TDF_EXIT TDF_SCHED1 /* thread is being killed. */ #define TDF_BOUND TDF_SCHED2 -#define ke_flags ke_thread->td_flags -#define KEF_DIDRUN TDF_DIDRUN /* KSE actually ran. */ -#define KEF_EXIT TDF_EXIT /* KSE is being killed. */ -#define KEF_BOUND TDF_BOUND /* stuck to one CPU */ +#define ts_flags ts_thread->td_flags +#define TSF_DIDRUN TDF_DIDRUN /* thread actually ran. */ +#define TSF_EXIT TDF_EXIT /* thread is being killed. */ +#define TSF_BOUND TDF_BOUND /* stuck to one CPU */ -#define SKE_RUNQ_PCPU(ke) \ - ((ke)->ke_runq != 0 && (ke)->ke_runq != &runq) +#define SKE_RUNQ_PCPU(ts) \ + ((ts)->ts_runq != 0 && (ts)->ts_runq != &runq) -#ifdef KSE -struct kg_sched { - struct thread *skg_last_assigned; /* (j) Last thread assigned to */ - /* the system scheduler. */ - int skg_avail_opennings; /* (j) Num KSEs requested in group. */ - int skg_concurrency; /* (j) Num KSEs requested in group. */ -}; -#define kg_last_assigned kg_sched->skg_last_assigned -#define kg_avail_opennings kg_sched->skg_avail_opennings -#define kg_concurrency kg_sched->skg_concurrency - -#define SLOT_RELEASE(kg) \ -do { \ - kg->kg_avail_opennings++; \ - CTR3(KTR_RUNQ, "kg %p(%d) Slot released (->%d)", \ - kg, \ - kg->kg_concurrency, \ - kg->kg_avail_opennings); \ -/* KASSERT((kg->kg_avail_opennings <= kg->kg_concurrency), \ - ("slots out of whack"));*/ \ -} while (0) - -#define SLOT_USE(kg) \ -do { \ - kg->kg_avail_opennings--; \ - CTR3(KTR_RUNQ, "kg %p(%d) Slot used (->%d)", \ - kg, \ - kg->kg_concurrency, \ - kg->kg_avail_opennings); \ -/* KASSERT((kg->kg_avail_opennings >= 0), \ - ("slots out of whack"));*/ \ -} while (0) -#endif - -/* - * KSE_CAN_MIGRATE macro returns true if the kse can migrate between - * cpus. - */ -#define KSE_CAN_MIGRATE(ke) \ - ((ke)->ke_thread->td_pinned == 0 && ((ke)->ke_flags & KEF_BOUND) == 0) - -static struct kse kse0; -#ifdef KSE -static struct kg_sched kg_sched0; -#endif +static struct td_sched td_sched0; static int sched_tdcnt; /* Total runnable threads in the system. */ static int sched_quantum; /* Roundrobin scheduling quantum in ticks. */ @@ -175,12 +112,7 @@ static int sched_quantum; /* Roundrobin scheduling quantum in ticks. */ static struct callout roundrobin_callout; -#ifdef KSE -static void slot_fill(struct ksegrp *kg); -static struct kse *sched_choose(void); /* XXX Should be thread * */ -#else -static struct thread *sched_choose(void); -#endif +static struct td_sched *sched_choose(void); static void setup_runqs(void); static void roundrobin(void *arg); @@ -189,15 +121,9 @@ static void schedcpu_thread(void); static void sched_priority(struct thread *td, u_char prio); static void sched_setup(void *dummy); static void maybe_resched(struct thread *td); -#ifdef KSE -static void updatepri(struct ksegrp *kg); -static void resetpriority(struct ksegrp *kg); -static void resetpriority_thread(struct thread *td, struct ksegrp *kg); -#else static void updatepri(struct thread *td); static void resetpriority(struct thread *td); static void resetpriority_thread(struct thread *td); -#endif #ifdef SMP static int forward_wakeup(int cpunum); #endif @@ -300,21 +226,11 @@ SYSCTL_INT(_kern_sched_ipiwakeup, OID_AUTO, htt2, CTLFLAG_RW, "account for htt"); #endif -#ifdef KSE +#if 0 static int sched_followon = 0; SYSCTL_INT(_kern_sched, OID_AUTO, followon, CTLFLAG_RW, &sched_followon, 0, "allow threads to share a quantum"); - -static int sched_pfollowons = 0; -SYSCTL_INT(_kern_sched, OID_AUTO, pfollowons, CTLFLAG_RD, - &sched_pfollowons, 0, - "number of followons done to a different ksegrp"); - -static int sched_kgfollowons = 0; -SYSCTL_INT(_kern_sched, OID_AUTO, kgfollowons, CTLFLAG_RD, - &sched_kgfollowons, 0, - "number of followons done in a ksegrp"); #endif static __inline void @@ -366,40 +282,20 @@ roundrobin(void *arg) /* * Constants for digital decay and forget: - * ifdef KSE - * 90% of (kg_estcpu) usage in 5 * loadav time - * else * 90% of (td_estcpu) usage in 5 * loadav time - * endif - * 95% of (ke_pctcpu) usage in 60 seconds (load insensitive) + * 95% of (ts_pctcpu) usage in 60 seconds (load insensitive) * Note that, as ps(1) mentions, this can let percentages * total over 100% (I've seen 137.9% for 3 processes). * - * ifdef KSE - * Note that schedclock() updates kg_estcpu and p_cpticks asynchronously. - * else * Note that schedclock() updates td_estcpu and p_cpticks asynchronously. - * endif * - * ifdef KSE - * We wish to decay away 90% of kg_estcpu in (5 * loadavg) seconds. - * else * We wish to decay away 90% of td_estcpu in (5 * loadavg) seconds. - * endif * That is, the system wants to compute a value of decay such * that the following for loop: * for (i = 0; i < (5 * loadavg); i++) - * ifdef KSE - * kg_estcpu *= decay; - * else * td_estcpu *= decay; - * endif * will compute - * ifdef KSE - * kg_estcpu *= 0.1; - * else * td_estcpu *= 0.1; - * endif * for all values of loadavg: * * Mathematically this loop can be expressed by saying: @@ -452,7 +348,7 @@ roundrobin(void *arg) #define loadfactor(loadav) (2 * (loadav)) #define decay_cpu(loadfac, cpu) (((loadfac) * (cpu)) / ((loadfac) + FSCALE)) -/* decay 95% of `ke_pctcpu' in 60 seconds; see CCPU_SHIFT before changing */ +/* decay 95% of `ts_pctcpu' in 60 seconds; see CCPU_SHIFT before changing */ static fixpt_t ccpu = 0.95122942450071400909 * FSCALE; /* exp(-1/20) */ SYSCTL_INT(_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0, ""); @@ -481,10 +377,7 @@ schedcpu(void) register fixpt_t loadfac = loadfactor(averunnable.ldavg[0]); struct thread *td; struct proc *p; - struct kse *ke; -#ifdef KSE - struct ksegrp *kg; -#endif + struct td_sched *ts; int awake, realstathz; realstathz = stathz ? stathz : hz; @@ -499,126 +392,63 @@ schedcpu(void) * 16-bit int's (remember them?) overflow takes 45 days. */ p->p_swtime++; -#ifdef KSE - FOREACH_KSEGRP_IN_PROC(p, kg) { -#else FOREACH_THREAD_IN_PROC(p, td) { -#endif awake = 0; -#ifdef KSE - FOREACH_THREAD_IN_GROUP(kg, td) { - ke = td->td_kse; - /* - * Increment sleep time (if sleeping). We - * ignore overflow, as above. - */ - /* - * The kse slptimes are not touched in wakeup - * because the thread may not HAVE a KSE. - */ - if (ke->ke_state == KES_ONRUNQ) { - awake = 1; - ke->ke_flags &= ~KEF_DIDRUN; - } else if ((ke->ke_state == KES_THREAD) && - (TD_IS_RUNNING(td))) { - awake = 1; - /* Do not clear KEF_DIDRUN */ - } else if (ke->ke_flags & KEF_DIDRUN) { - awake = 1; - ke->ke_flags &= ~KEF_DIDRUN; - } - - /* - * ke_pctcpu is only for ps and ttyinfo(). - * Do it per kse, and add them up at the end? - * XXXKSE - */ - ke->ke_pctcpu = (ke->ke_pctcpu * ccpu) >> - FSHIFT; - /* - * If the kse has been idle the entire second, - * stop recalculating its priority until - * it wakes up. - */ - if (ke->ke_cpticks == 0) - continue; -#if (FSHIFT >= CCPU_SHIFT) - ke->ke_pctcpu += (realstathz == 100) - ? ((fixpt_t) ke->ke_cpticks) << - (FSHIFT - CCPU_SHIFT) : - 100 * (((fixpt_t) ke->ke_cpticks) - << (FSHIFT - CCPU_SHIFT)) / realstathz; -#else - ke->ke_pctcpu += ((FSCALE - ccpu) * - (ke->ke_cpticks * - FSCALE / realstathz)) >> FSHIFT; -#endif - ke->ke_cpticks = 0; - } /* end of kse loop */ -#else - ke = td->td_kse; + ts = td->td_sched; /* * Increment sleep time (if sleeping). We * ignore overflow, as above. */ /* - * The kse slptimes are not touched in wakeup - * because the thread may not HAVE a KSE. + * The td_sched slptimes are not touched in wakeup + * because the thread may not HAVE everything in + * memory? XXX I think this is out of date. */ - if (ke->ke_state == KES_ONRUNQ) { + if (ts->ts_state == TSS_ONRUNQ) { awake = 1; - ke->ke_flags &= ~KEF_DIDRUN; - } else if ((ke->ke_state == KES_THREAD) && + ts->ts_flags &= ~TSF_DIDRUN; + } else if ((ts->ts_state == TSS_THREAD) && (TD_IS_RUNNING(td))) { awake = 1; - /* Do not clear KEF_DIDRUN */ - } else if (ke->ke_flags & KEF_DIDRUN) { + /* Do not clear TSF_DIDRUN */ + } else if (ts->ts_flags & TSF_DIDRUN) { awake = 1; - ke->ke_flags &= ~KEF_DIDRUN; + ts->ts_flags &= ~TSF_DIDRUN; } /* - * ke_pctcpu is only for ps and ttyinfo(). - * Do it per kse, and add them up at the end? + * ts_pctcpu is only for ps and ttyinfo(). + * Do it per td_sched, and add them up at the end? * XXXKSE */ - ke->ke_pctcpu = (ke->ke_pctcpu * ccpu) >> - FSHIFT; + ts->ts_pctcpu = (ts->ts_pctcpu * ccpu) >> FSHIFT; /* - * If the kse has been idle the entire second, + * If the td_sched has been idle the entire second, * stop recalculating its priority until * it wakes up. */ - if (ke->ke_cpticks != 0) { + if (ts->ts_cpticks != 0) { #if (FSHIFT >= CCPU_SHIFT) - ke->ke_pctcpu += (realstathz == 100) - ? ((fixpt_t) ke->ke_cpticks) << - (FSHIFT - CCPU_SHIFT) : - 100 * (((fixpt_t) ke->ke_cpticks) - << (FSHIFT - CCPU_SHIFT)) / realstathz; + ts->ts_pctcpu += (realstathz == 100) + ? ((fixpt_t) ts->ts_cpticks) << + (FSHIFT - CCPU_SHIFT) : + 100 * (((fixpt_t) ts->ts_cpticks) + << (FSHIFT - CCPU_SHIFT)) / realstathz; #else - ke->ke_pctcpu += ((FSCALE - ccpu) * - (ke->ke_cpticks * - FSCALE / realstathz)) >> FSHIFT; + ts->ts_pctcpu += ((FSCALE - ccpu) * + (ts->ts_cpticks * + FSCALE / realstathz)) >> FSHIFT; #endif - ke->ke_cpticks = 0; + ts->ts_cpticks = 0; } -#endif - /* - * ifdef KSE - * If there are ANY running threads in this KSEGRP, - * else * If there are ANY running threads in this process, - * endif * then don't count it as sleeping. +XXX this is broken + */ if (awake) { -#ifdef KSE - if (kg->kg_slptime > 1) { -#else - if (td->td_slptime > 1) { -#endif + if (p->p_slptime > 1) { /* * In an ideal world, this should not * happen, because whoever woke us @@ -628,21 +458,6 @@ schedcpu(void) * priority. Should KASSERT at some * point when all the cases are fixed. */ -#ifdef KSE - updatepri(kg); - } - kg->kg_slptime = 0; - } else - kg->kg_slptime++; - if (kg->kg_slptime > 1) - continue; - kg->kg_estcpu = decay_cpu(loadfac, kg->kg_estcpu); - resetpriority(kg); - FOREACH_THREAD_IN_GROUP(kg, td) { - resetpriority_thread(td, kg); - } - } /* end of ksegrp loop */ -#else updatepri(td); } td->td_slptime = 0; @@ -654,7 +469,6 @@ schedcpu(void) resetpriority(td); resetpriority_thread(td); } /* end of thread loop */ -#endif mtx_unlock_spin(&sched_lock); } /* end of process loop */ sx_sunlock(&allproc_lock); @@ -676,48 +490,24 @@ schedcpu_thread(void) /* * Recalculate the priority of a process after it has slept for a while. - * ifdef KSE - * For all load averages >= 1 and max kg_estcpu of 255, sleeping for at - * least six times the loadfactor will decay kg_estcpu to zero. - * else * For all load averages >= 1 and max td_estcpu of 255, sleeping for at * least six times the loadfactor will decay td_estcpu to zero. - * endif */ static void -#ifdef KSE -updatepri(struct ksegrp *kg) -#else updatepri(struct thread *td) -#endif { register fixpt_t loadfac; register unsigned int newcpu; loadfac = loadfactor(averunnable.ldavg[0]); -#ifdef KSE - if (kg->kg_slptime > 5 * loadfac) - kg->kg_estcpu = 0; -#else if (td->td_slptime > 5 * loadfac) td->td_estcpu = 0; -#endif else { -#ifdef KSE - newcpu = kg->kg_estcpu; - kg->kg_slptime--; /* was incremented in schedcpu() */ - while (newcpu && --kg->kg_slptime) -#else newcpu = td->td_estcpu; td->td_slptime--; /* was incremented in schedcpu() */ while (newcpu && --td->td_slptime) -#endif newcpu = decay_cpu(loadfac, newcpu); -#ifdef KSE - kg->kg_estcpu = newcpu; -#else td->td_estcpu = newcpu; -#endif } } @@ -727,43 +517,25 @@ updatepri(struct thread *td) * than that of the current process. */ static void -#ifdef KSE -resetpriority(struct ksegrp *kg) -#else resetpriority(struct thread *td) -#endif { register unsigned int newpriority; -#ifdef KSE - if (kg->kg_pri_class == PRI_TIMESHARE) { - newpriority = PUSER + kg->kg_estcpu / INVERSE_ESTCPU_WEIGHT + - NICE_WEIGHT * (kg->kg_proc->p_nice - PRIO_MIN); -#else if (td->td_pri_class == PRI_TIMESHARE) { newpriority = PUSER + td->td_estcpu / INVERSE_ESTCPU_WEIGHT + NICE_WEIGHT * (td->td_proc->p_nice - PRIO_MIN); -#endif newpriority = min(max(newpriority, PRI_MIN_TIMESHARE), PRI_MAX_TIMESHARE); -#ifdef KSE - sched_user_prio(kg, newpriority); -#else sched_user_prio(td, newpriority); -#endif } } /* - * Update the thread's priority when the associated ksegroup's user + * Update the thread's priority when the associated process's user * priority changes. */ static void -#ifdef KSE -resetpriority_thread(struct thread *td, struct ksegrp *kg) -#else resetpriority_thread(struct thread *td) -#endif { /* Only change threads with a time sharing user priority. */ @@ -774,11 +546,7 @@ resetpriority_thread(struct thread *td) /* XXX the whole needresched thing is broken, but not silly. */ maybe_resched(td); -#ifdef KSE - sched_prio(td, kg->kg_user_pri); -#else sched_prio(td, td->td_user_pri); -#endif } /* ARGSUSED */ @@ -814,16 +582,9 @@ schedinit(void) * Set up the scheduler specific parts of proc0. */ proc0.p_sched = NULL; /* XXX */ -#ifdef KSE - ksegrp0.kg_sched = &kg_sched0; -#endif - thread0.td_sched = &kse0; - kse0.ke_thread = &thread0; - kse0.ke_state = KES_THREAD; -#ifdef KSE - kg_sched0.skg_concurrency = 1; - kg_sched0.skg_avail_opennings = 0; /* we are already running */ -#endif + thread0.td_sched = &td_sched0; + td_sched0.ts_thread = &thread0; + td_sched0.ts_state = TSS_THREAD; } int @@ -847,13 +608,8 @@ sched_rr_interval(void) /* * We adjust the priority of the current process. The priority of * a process gets worse as it accumulates CPU time. The cpu usage - * ifdef KSE - * estimator (kg_estcpu) is increased here. resetpriority() will - * compute a different priority each time kg_estcpu increases by - * else * estimator (td_estcpu) is increased here. resetpriority() will * compute a different priority each time td_estcpu increases by - * endif * INVERSE_ESTCPU_WEIGHT * (until MAXPRI is reached). The cpu usage estimator ramps up * quite quickly when the process is running (linearly), and decays @@ -866,163 +622,83 @@ sched_rr_interval(void) void sched_clock(struct thread *td) { -#ifdef KSE - struct ksegrp *kg; -#endif - struct kse *ke; + struct td_sched *ts; mtx_assert(&sched_lock, MA_OWNED); -#ifdef KSE - kg = td->td_ksegrp; -#endif - ke = td->td_kse; + ts = td->td_sched; - ke->ke_cpticks++; -#ifdef KSE - kg->kg_estcpu = ESTCPULIM(kg->kg_estcpu + 1); - if ((kg->kg_estcpu % INVERSE_ESTCPU_WEIGHT) == 0) { - resetpriority(kg); - resetpriority_thread(td, kg); -#else + ts->ts_cpticks++; td->td_estcpu = ESTCPULIM(td->td_estcpu + 1); if ((td->td_estcpu % INVERSE_ESTCPU_WEIGHT) == 0) { resetpriority(td); resetpriority_thread(td); -#endif } } -#ifdef KSE -/* - * charge childs scheduling cpu usage to parent. - * - * XXXKSE assume only one thread & kse & ksegrp keep estcpu in each ksegrp. - * Charge it to the ksegrp that did the wait since process estcpu is sum of - * all ksegrps, this is strictly as expected. Assume that the child process - * aggregated all the estcpu into the 'built-in' ksegrp. - */ -#else /* * charge childs scheduling cpu usage to parent. */ -#endif void sched_exit(struct proc *p, struct thread *td) { -#ifdef KSE - sched_exit_ksegrp(FIRST_KSEGRP_IN_PROC(p), td); - sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); -#else - struct thread *parent = FIRST_THREAD_IN_PROC(p); CTR3(KTR_SCHED, "sched_exit: %p(%s) prio %d", td, td->td_proc->p_comm, td->td_priority); - parent->td_estcpu = ESTCPULIM(parent->td_estcpu + td->td_estcpu); - if ((td->td_proc->p_flag & P_NOLOAD) == 0) - sched_load_rem(); -#endif -} - -#ifdef KSE -void -sched_exit_ksegrp(struct ksegrp *kg, struct thread *childtd) -{ - - mtx_assert(&sched_lock, MA_OWNED); - kg->kg_estcpu = ESTCPULIM(kg->kg_estcpu + childtd->td_ksegrp->kg_estcpu); + sched_exit_thread(FIRST_THREAD_IN_PROC(p), td); } void sched_exit_thread(struct thread *td, struct thread *child) { + struct proc *childproc = child->td_proc; + CTR3(KTR_SCHED, "sched_exit_thread: %p(%s) prio %d", - child, child->td_proc->p_comm, child->td_priority); + child, childproc->p_comm, child->td_priority); + td->td_estcpu = ESTCPULIM(td->td_estcpu + child->td_estcpu); + childproc->p_estcpu = ESTCPULIM(childproc->p_estcpu + + child->td_estcpu); if ((child->td_proc->p_flag & P_NOLOAD) == 0) sched_load_rem(); } -#endif void sched_fork(struct thread *td, struct thread *childtd) { -#ifdef KSE - sched_fork_ksegrp(td, childtd->td_ksegrp); sched_fork_thread(td, childtd); -#else - childtd->td_estcpu = td->td_estcpu; - sched_newthread(childtd); -#endif -} - -#ifdef KSE -void -sched_fork_ksegrp(struct thread *td, struct ksegrp *child) -{ - mtx_assert(&sched_lock, MA_OWNED); - child->kg_estcpu = td->td_ksegrp->kg_estcpu; } void sched_fork_thread(struct thread *td, struct thread *childtd) { + childtd->td_estcpu = td->td_estcpu; sched_newthread(childtd); } -#endif void sched_nice(struct proc *p, int nice) { -#ifdef KSE - struct ksegrp *kg; -#endif struct thread *td; PROC_LOCK_ASSERT(p, MA_OWNED); mtx_assert(&sched_lock, MA_OWNED); p->p_nice = nice; -#ifdef KSE - FOREACH_KSEGRP_IN_PROC(p, kg) { - resetpriority(kg); - FOREACH_THREAD_IN_GROUP(kg, td) { - resetpriority_thread(td, kg); - } - } -#else FOREACH_THREAD_IN_PROC(p, td) { resetpriority(td); resetpriority_thread(td); } -#endif } void -#ifdef KSE -sched_class(struct ksegrp *kg, int class) -#else sched_class(struct thread *td, int class) -#endif { mtx_assert(&sched_lock, MA_OWNED); -#ifdef KSE - kg->kg_pri_class = class; -#else td->td_pri_class = class; -#endif } -#ifdef KSE -/* - * Adjust the priority of a thread. - * This may include moving the thread within the KSEGRP, - * changing the assignment of a kse to the thread, - * and moving a KSE in the system run queue. - */ -#else /* * Adjust the priority of a thread. */ -#endif static void sched_priority(struct thread *td, u_char prio) { @@ -1067,11 +743,7 @@ sched_unlend_prio(struct thread *td, u_char prio) if (td->td_base_pri >= PRI_MIN_TIMESHARE && td->td_base_pri <= PRI_MAX_TIMESHARE) -#ifdef KSE - base_pri = td->td_ksegrp->kg_user_pri; -#else base_pri = td->td_user_pri; -#endif else base_pri = td->td_base_pri; if (prio >= base_pri) { @@ -1109,40 +781,15 @@ sched_prio(struct thread *td, u_char prio) } void -#ifdef KSE -sched_user_prio(struct ksegrp *kg, u_char prio) -#else sched_user_prio(struct thread *td, u_char prio) -#endif { -#ifdef KSE - struct thread *td; -#endif u_char oldprio; -#ifdef KSE - kg->kg_base_user_pri = prio; - - /* XXXKSE only for 1:1 */ - - td = TAILQ_FIRST(&kg->kg_threads); - if (td == NULL) { - kg->kg_user_pri = prio; - return; - } - - if (td->td_flags & TDF_UBORROWING && kg->kg_user_pri <= prio) - return; - - oldprio = kg->kg_user_pri; - kg->kg_user_pri = prio; -#else td->td_base_user_pri = prio; if (td->td_flags & TDF_UBORROWING && td->td_user_pri <= prio) return; oldprio = td->td_user_pri; td->td_user_pri = prio; -#endif if (TD_ON_UPILOCK(td) && oldprio != prio) umtx_pi_adjust(td, oldprio); @@ -1155,13 +802,8 @@ sched_lend_user_prio(struct thread *td, u_char prio) td->td_flags |= TDF_UBORROWING; -#ifdef KSE - oldprio = td->td_ksegrp->kg_user_pri; - td->td_ksegrp->kg_user_pri = prio; -#else oldprio = td->td_user_pri; td->td_user_pri = prio; -#endif if (TD_ON_UPILOCK(td) && oldprio != prio) umtx_pi_adjust(td, oldprio); @@ -1170,23 +812,12 @@ sched_lend_user_prio(struct thread *td, u_char prio) void sched_unlend_user_prio(struct thread *td, u_char prio) { -#ifdef KSE - struct ksegrp *kg = td->td_ksegrp; -#endif u_char base_pri; -#ifdef KSE - base_pri = kg->kg_base_user_pri; -#else base_pri = td->td_base_user_pri; -#endif if (prio >= base_pri) { td->td_flags &= ~TDF_UBORROWING; -#ifdef KSE - sched_user_prio(kg, base_pri); -#else sched_user_prio(td, base_pri); -#endif } else sched_lend_user_prio(td, prio); } @@ -1196,59 +827,37 @@ sched_sleep(struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); -#ifdef KSE - td->td_ksegrp->kg_slptime = 0; -#else td->td_slptime = 0; -#endif } -#ifdef KSE -static void remrunqueue(struct thread *td); -#endif - void sched_switch(struct thread *td, struct thread *newtd, int flags) { - struct kse *ke; -#ifdef KSE - struct ksegrp *kg; -#endif + struct td_sched *ts; struct proc *p; - ke = td->td_kse; + ts = td->td_sched; p = td->td_proc; mtx_assert(&sched_lock, MA_OWNED); if ((p->p_flag & P_NOLOAD) == 0) sched_load_rem(); -#ifdef KSE +#if 0 /* * We are volunteering to switch out so we get to nominate * a successor for the rest of our quantum - * First try another thread in our ksegrp, and then look for - * other ksegrps in our process. + * First try another thread in our process + * + * this is too expensive to do without per process run queues + * so skip it for now. + * XXX keep this comment as a marker. */ if (sched_followon && (p->p_flag & P_HADTHREADS) && (flags & SW_VOL) && - newtd == NULL) { - /* lets schedule another thread from this process */ - kg = td->td_ksegrp; - if ((newtd = TAILQ_FIRST(&kg->kg_runq))) { - remrunqueue(newtd); - sched_kgfollowons++; - } else { - FOREACH_KSEGRP_IN_PROC(p, kg) { - if ((newtd = TAILQ_FIRST(&kg->kg_runq))) { - sched_pfollowons++; - remrunqueue(newtd); - break; - } - } - } - } + newtd == NULL) + newtd = mumble(); #endif if (newtd) @@ -1267,25 +876,11 @@ sched_switch(struct thread *td, struct thread *newtd, int flags) if (td == PCPU_GET(idlethread)) TD_SET_CAN_RUN(td); else { -#ifdef KSE - SLOT_RELEASE(td->td_ksegrp); -#endif if (TD_IS_RUNNING(td)) { - /* Put us back on the run queue (kse and all). */ + /* Put us back on the run queue. */ setrunqueue(td, (flags & SW_PREEMPT) ? SRQ_OURSELF|SRQ_YIELDING|SRQ_PREEMPTED : SRQ_OURSELF|SRQ_YIELDING); -#ifdef KSE - } else if (p->p_flag & P_HADTHREADS) { - /* - * We will not be on the run queue. So we must be - * sleeping or similar. As it's available, - * someone else can use the KSE if they need it. - * It's NOT available if we are about to need it - */ - if (newtd == NULL || newtd->td_ksegrp != td->td_ksegrp) - slot_fill(td->td_ksegrp); -#endif } } if (newtd) { @@ -1294,17 +889,12 @@ sched_switch(struct thread *td, struct thread *newtd, int flags) * as if it had been added to the run queue and selected. * It came from: * * A preemption - * ifdef KSE * * An upcall - * endif * * A followon */ KASSERT((newtd->td_inhibitors == 0), ("trying to run inhibitted thread")); -#ifdef KSE - SLOT_USE(newtd->td_ksegrp); -#endif - newtd->td_kse->ke_flags |= KEF_DIDRUN; + newtd->td_sched->ts_flags |= TSF_DIDRUN; TD_SET_RUNNING(newtd); if ((newtd->td_proc->p_flag & P_NOLOAD) == 0) sched_load_add(); @@ -1332,25 +922,12 @@ sched_switch(struct thread *td, struct thread *newtd, int flags) void sched_wakeup(struct thread *td) { -#ifdef KSE - struct ksegrp *kg; -#endif - mtx_assert(&sched_lock, MA_OWNED); -#ifdef KSE - kg = td->td_ksegrp; - if (kg->kg_slptime > 1) { - updatepri(kg); - resetpriority(kg); - } - kg->kg_slptime = 0; -#else if (td->td_slptime > 1) { updatepri(td); resetpriority(td); } td->td_slptime = 0; -#endif setrunqueue(td, SRQ_BORING); } @@ -1481,22 +1058,17 @@ void sched_add(struct thread *td, int flags) #ifdef SMP { - struct kse *ke; + struct td_sched *ts; int forwarded = 0; int cpu; int single_cpu = 0; - ke = td->td_kse; + ts = td->td_sched; mtx_assert(&sched_lock, MA_OWNED); - KASSERT(ke->ke_state != KES_ONRUNQ, - ("sched_add: kse %p (%s) already in run queue", ke, -#ifdef KSE - ke->ke_proc->p_comm)); - KASSERT(ke->ke_proc->p_sflag & PS_INMEM, -#else + KASSERT(ts->ts_state != TSS_ONRUNQ, + ("sched_add: td_sched %p (%s) already in run queue", ts, td->td_proc->p_comm)); KASSERT(td->td_proc->p_sflag & PS_INMEM, -#endif ("sched_add: process swapped out")); CTR5(KTR_SCHED, "sched_add: %p(%s) prio %d by %p(%s)", td, td->td_proc->p_comm, td->td_priority, curthread, @@ -1505,22 +1077,22 @@ sched_add(struct thread *td, int flags) if (td->td_pinned != 0) { cpu = td->td_lastcpu; - ke->ke_runq = &runq_pcpu[cpu]; + ts->ts_runq = &runq_pcpu[cpu]; single_cpu = 1; CTR3(KTR_RUNQ, - "sched_add: Put kse:%p(td:%p) on cpu%d runq", ke, td, cpu); - } else if ((ke)->ke_flags & KEF_BOUND) { + "sched_add: Put td_sched:%p(td:%p) on cpu%d runq", ts, td, cpu); + } else if ((ts)->ts_flags & TSF_BOUND) { /* Find CPU from bound runq */ - KASSERT(SKE_RUNQ_PCPU(ke),("sched_add: bound kse not on cpu runq")); - cpu = ke->ke_runq - &runq_pcpu[0]; + KASSERT(SKE_RUNQ_PCPU(ts),("sched_add: bound td_sched not on cpu runq")); + cpu = ts->ts_runq - &runq_pcpu[0]; single_cpu = 1; CTR3(KTR_RUNQ, - "sched_add: Put kse:%p(td:%p) on cpu%d runq", ke, td, cpu); + "sched_add: Put td_sched:%p(td:%p) on cpu%d runq", ts, td, cpu); } else { CTR2(KTR_RUNQ, - "sched_add: adding kse:%p (td:%p) to gbl runq", ke, td); + "sched_add: adding td_sched:%p (td:%p) to gbl runq", ts, td); cpu = NOCPU; - ke->ke_runq = &runq; + ts->ts_runq = &runq; } if (single_cpu && (cpu != PCPU_GET(cpuid))) { @@ -1546,32 +1118,24 @@ sched_add(struct thread *td, int flags) if ((td->td_proc->p_flag & P_NOLOAD) == 0) sched_load_add(); -#ifdef KSE - SLOT_USE(td->td_ksegrp); -#endif - runq_add(ke->ke_runq, ke, flags); - ke->ke_state = KES_ONRUNQ; + runq_add(ts->ts_runq, ts, flags); + ts->ts_state = TSS_ONRUNQ; } #else /* SMP */ { - struct kse *ke; - ke = td->td_kse; + struct td_sched *ts; + ts = td->td_sched; mtx_assert(&sched_lock, MA_OWNED); - KASSERT(ke->ke_state != KES_ONRUNQ, - ("sched_add: kse %p (%s) already in run queue", ke, -#ifdef KSE - ke->ke_proc->p_comm)); - KASSERT(ke->ke_proc->p_sflag & PS_INMEM, -#else + KASSERT(ts->ts_state != TSS_ONRUNQ, + ("sched_add: td_sched %p (%s) already in run queue", ts, td->td_proc->p_comm)); KASSERT(td->td_proc->p_sflag & PS_INMEM, -#endif ("sched_add: process swapped out")); CTR5(KTR_SCHED, "sched_add: %p(%s) prio %d by %p(%s)", td, td->td_proc->p_comm, td->td_priority, curthread, curthread->td_proc->p_comm); - CTR2(KTR_RUNQ, "sched_add: adding kse:%p (td:%p) to runq", ke, td); - ke->ke_runq = &runq; + CTR2(KTR_RUNQ, "sched_add: adding td_sched:%p (td:%p) to runq", ts, td); + ts->ts_runq = &runq; /* * If we are yielding (on the way out anyhow) @@ -1590,11 +1154,8 @@ sched_add(struct thread *td, int flags) } if ((td->td_proc->p_flag & P_NOLOAD) == 0) sched_load_add(); -#ifdef KSE - SLOT_USE(td->td_ksegrp); -#endif - runq_add(ke->ke_runq, ke, flags); - ke->ke_state = KES_ONRUNQ; + runq_add(ts->ts_runq, ts, flags); + ts->ts_state = TSS_ONRUNQ; maybe_resched(td); } #endif /* SMP */ @@ -1602,17 +1163,13 @@ sched_add(struct thread *td, int flags) void sched_rem(struct thread *td) { - struct kse *ke; + struct td_sched *ts; - ke = td->td_kse; -#ifdef KSE - KASSERT(ke->ke_proc->p_sflag & PS_INMEM, -#else + ts = td->td_sched; KASSERT(td->td_proc->p_sflag & PS_INMEM, -#endif ("sched_rem: process swapped out")); - KASSERT((ke->ke_state == KES_ONRUNQ), - ("sched_rem: KSE not on run queue")); + KASSERT((ts->ts_state == TSS_ONRUNQ), + ("sched_rem: thread not on run queue")); mtx_assert(&sched_lock, MA_OWNED); CTR5(KTR_SCHED, "sched_rem: %p(%s) prio %d by %p(%s)", td, td->td_proc->p_comm, td->td_priority, curthread, @@ -1620,81 +1177,57 @@ sched_rem(struct thread *td) if ((td->td_proc->p_flag & P_NOLOAD) == 0) sched_load_rem(); -#ifdef KSE - SLOT_RELEASE(td->td_ksegrp); -#endif - runq_remove(ke->ke_runq, ke); + runq_remove(ts->ts_runq, ts); - ke->ke_state = KES_THREAD; + ts->ts_state = TSS_THREAD; } /* * Select threads to run. * Notice that the running threads still consume a slot. */ -#ifdef KSE -struct kse * -#else -struct thread * -#endif +struct td_sched * sched_choose(void) { - struct kse *ke; + struct td_sched *ts; struct runq *rq; #ifdef SMP - struct kse *kecpu; + struct td_sched *kecpu; rq = &runq; - ke = runq_choose(&runq); + ts = runq_choose(&runq); kecpu = runq_choose(&runq_pcpu[PCPU_GET(cpuid)]); - if (ke == NULL || + if (ts == NULL || (kecpu != NULL && - kecpu->ke_thread->td_priority < ke->ke_thread->td_priority)) { - CTR2(KTR_RUNQ, "choosing kse %p from pcpu runq %d", kecpu, + kecpu->ts_thread->td_priority < ts->ts_thread->td_priority)) { + CTR2(KTR_RUNQ, "choosing td_sched %p from pcpu runq %d", kecpu, PCPU_GET(cpuid)); - ke = kecpu; + ts = kecpu; rq = &runq_pcpu[PCPU_GET(cpuid)]; } else { - CTR1(KTR_RUNQ, "choosing kse %p from main runq", ke); + CTR1(KTR_RUNQ, "choosing td_sched %p from main runq", ts); } #else rq = &runq; - ke = runq_choose(&runq); + ts = runq_choose(&runq); #endif -#ifdef KSE - if (ke != NULL) { -#else - if (ke) { -#endif - runq_remove(rq, ke); - ke->ke_state = KES_THREAD; + if (ts) { + runq_remove(rq, ts); + ts->ts_state = TSS_THREAD; -#ifdef KSE - KASSERT(ke->ke_proc->p_sflag & PS_INMEM, + KASSERT(ts->ts_thread->td_proc->p_sflag & PS_INMEM, ("sched_choose: process swapped out")); -#else - KASSERT(ke->ke_thread->td_proc->p_sflag & PS_INMEM, - ("sched_choose: process swapped out")); - return (ke->ke_thread); -#endif } -#ifdef KSE - return (ke); -#else - return (NULL); -#endif + return (ts); } void sched_userret(struct thread *td) { -#ifdef KSE - struct ksegrp *kg; -#endif /* * XXX we cheat slightly on the locking here to avoid locking in * the usual case. Setting td_priority here is essentially an @@ -1706,42 +1239,32 @@ sched_userret(struct thread *td) */ KASSERT((td->td_flags & TDF_BORROWING) == 0, ("thread with borrowed priority returning to userland")); -#ifdef KSE - kg = td->td_ksegrp; - if (td->td_priority != kg->kg_user_pri) { - mtx_lock_spin(&sched_lock); - td->td_priority = kg->kg_user_pri; - td->td_base_pri = kg->kg_user_pri; - mtx_unlock_spin(&sched_lock); - } -#else if (td->td_priority != td->td_user_pri) { mtx_lock_spin(&sched_lock); td->td_priority = td->td_user_pri; td->td_base_pri = td->td_user_pri; mtx_unlock_spin(&sched_lock); } -#endif } void sched_bind(struct thread *td, int cpu) { - struct kse *ke; + struct td_sched *ts; mtx_assert(&sched_lock, MA_OWNED); KASSERT(TD_IS_RUNNING(td), ("sched_bind: cannot bind non-running thread")); - ke = td->td_kse; + ts = td->td_sched; - ke->ke_flags |= KEF_BOUND; + ts->ts_flags |= TSF_BOUND; #ifdef SMP - ke->ke_runq = &runq_pcpu[cpu]; + ts->ts_runq = &runq_pcpu[cpu]; if (PCPU_GET(cpuid) == cpu) return; - ke->ke_state = KES_THREAD; + ts->ts_state = TSS_THREAD; mi_switch(SW_VOL, NULL); #endif @@ -1751,30 +1274,21 @@ void sched_unbind(struct thread* td) { mtx_assert(&sched_lock, MA_OWNED); - td->td_kse->ke_flags &= ~KEF_BOUND; + td->td_sched->ts_flags &= ~TSF_BOUND; } int sched_is_bound(struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); - return (td->td_kse->ke_flags & KEF_BOUND); + return (td->td_sched->ts_flags & TSF_BOUND); } void sched_relinquish(struct thread *td) { -#ifdef KSE - struct ksegrp *kg; - - kg = td->td_ksegrp; -#endif mtx_lock_spin(&sched_lock); -#ifdef KSE - if (kg->kg_pri_class == PRI_TIMESHARE) -#else if (td->td_pri_class == PRI_TIMESHARE) -#endif sched_prio(td, PRI_MAX_TIMESHARE); mi_switch(SW_VOL, NULL); mtx_unlock_spin(&sched_lock); @@ -1786,14 +1300,6 @@ sched_load(void) return (sched_tdcnt); } -#ifdef KSE -int -sched_sizeof_ksegrp(void) -{ - return (sizeof(struct ksegrp) + sizeof(struct kg_sched)); -} -#endif - int sched_sizeof_proc(void) { @@ -1803,16 +1309,16 @@ sched_sizeof_proc(void) int sched_sizeof_thread(void) { - return (sizeof(struct thread) + sizeof(struct kse)); + return (sizeof(struct thread) + sizeof(struct td_sched)); } fixpt_t sched_pctcpu(struct thread *td) { - struct kse *ke; + struct td_sched *ts; - ke = td->td_kse; - return (ke->ke_pctcpu); + ts = td->td_sched; + return (ts->ts_pctcpu); } void diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c index 7afe0fb5dd83..98b867bd96a4 100644 --- a/sys/kern/sched_ule.c +++ b/sys/kern/sched_ule.c @@ -30,8 +30,6 @@ __FBSDID("$FreeBSD$"); #include "opt_hwpmc_hooks.h" #include "opt_sched.h" -#define kse td_sched - #include #include #include @@ -92,46 +90,45 @@ int tickincr = 1 << 10; * but are scheduler specific. */ /* - * The schedulable entity that can be given a context to run. A process may - * have several of these. + * Thread scheduler specific section. + * fields int he thread structure that are specific to this scheduler. */ -struct td_sched { /* really kse */ - TAILQ_ENTRY(kse) ke_procq; /* (j/z) Run queue. */ - int ke_flags; /* (j) KEF_* flags. */ - struct thread *ke_thread; /* (*) Active associated thread. */ - fixpt_t ke_pctcpu; /* (j) %cpu during p_swtime. */ - u_char ke_rqindex; /* (j) Run queue index. */ +struct td_sched { + TAILQ_ENTRY(td_sched) ts_procq; /* (j/z) Run queue. */ + int ts_flags; /* (j) TSF_* flags. */ + struct thread *ts_thread; /* (*) Active associated thread. */ + fixpt_t ts_pctcpu; /* (j) %cpu during p_swtime. */ + u_char ts_rqindex; /* (j) Run queue index. */ enum { - KES_THREAD = 0x0, /* slaved to thread state */ - KES_ONRUNQ - } ke_state; /* (j) thread sched specific status. */ - int ke_slptime; - int ke_slice; - struct runq *ke_runq; - u_char ke_cpu; /* CPU that we have affinity for. */ + TSS_THREAD = 0x0, /* slaved to thread state */ + TSS_ONRUNQ + } ts_state; /* (j) thread sched specific status. */ + int ts_slptime; + int ts_slice; + struct runq *ts_runq; + u_char ts_cpu; /* CPU that we have affinity for. */ /* The following variables are only used for pctcpu calculation */ - int ke_ltick; /* Last tick that we were running on */ - int ke_ftick; /* First tick that we were running on */ - int ke_ticks; /* Tick count */ + int ts_ltick; /* Last tick that we were running on */ + int ts_ftick; /* First tick that we were running on */ + int ts_ticks; /* Tick count */ /* originally from kg_sched */ int skg_slptime; /* Number of ticks we vol. slept */ int skg_runtime; /* Number of ticks we were running */ }; -#define td_kse td_sched -#define ke_assign ke_procq.tqe_next -/* flags kept in ke_flags */ -#define KEF_ASSIGNED 0x0001 /* Thread is being migrated. */ -#define KEF_BOUND 0x0002 /* Thread can not migrate. */ -#define KEF_XFERABLE 0x0004 /* Thread was added as transferable. */ -#define KEF_HOLD 0x0008 /* Thread is temporarily bound. */ -#define KEF_REMOVED 0x0010 /* Thread was removed while ASSIGNED */ -#define KEF_INTERNAL 0x0020 /* Thread added due to migration. */ -#define KEF_PREEMPTED 0x0040 /* Thread was preempted */ -#define KEF_DIDRUN 0x02000 /* Thread actually ran. */ -#define KEF_EXIT 0x04000 /* Thread is being killed. */ +#define ts_assign ts_procq.tqe_next +/* flags kept in ts_flags */ +#define TSF_ASSIGNED 0x0001 /* Thread is being migrated. */ +#define TSF_BOUND 0x0002 /* Thread can not migrate. */ +#define TSF_XFERABLE 0x0004 /* Thread was added as transferable. */ +#define TSF_HOLD 0x0008 /* Thread is temporarily bound. */ +#define TSF_REMOVED 0x0010 /* Thread was removed while ASSIGNED */ +#define TSF_INTERNAL 0x0020 /* Thread added due to migration. */ +#define TSF_PREEMPTED 0x0040 /* Thread was preempted */ +#define TSF_DIDRUN 0x02000 /* Thread actually ran. */ +#define TSF_EXIT 0x04000 /* Thread is being killed. */ -static struct kse kse0; +static struct td_sched td_sched0; /* * The priority is primarily determined by the interactivity score. Thus, we @@ -191,9 +188,9 @@ static struct kse kse0; */ #define SCHED_INTERACTIVE(td) \ (sched_interact_score(td) < SCHED_INTERACT_THRESH) -#define SCHED_CURR(td, ke) \ - ((ke->ke_thread->td_flags & TDF_BORROWING) || \ - (ke->ke_flags & KEF_PREEMPTED) || SCHED_INTERACTIVE(td)) +#define SCHED_CURR(td, ts) \ + ((ts->ts_thread->td_flags & TDF_BORROWING) || \ + (ts->ts_flags & TSF_PREEMPTED) || SCHED_INTERACTIVE(td)) /* * Cpu percentage computation macros and defines. @@ -206,22 +203,22 @@ static struct kse kse0; #define SCHED_CPU_TICKS (hz * SCHED_CPU_TIME) /* - * kseq - per processor runqs and statistics. + * tdq - per processor runqs and statistics. */ -struct kseq { +struct tdq { struct runq ksq_idle; /* Queue of IDLE threads. */ struct runq ksq_timeshare[2]; /* Run queues for !IDLE. */ struct runq *ksq_next; /* Next timeshare queue. */ struct runq *ksq_curr; /* Current queue. */ int ksq_load_timeshare; /* Load for timeshare. */ int ksq_load; /* Aggregate load. */ - short ksq_nice[SCHED_PRI_NRESV]; /* KSEs in each nice bin. */ + short ksq_nice[SCHED_PRI_NRESV]; /* threadss in each nice bin. */ short ksq_nicemin; /* Least nice. */ #ifdef SMP int ksq_transferable; - LIST_ENTRY(kseq) ksq_siblings; /* Next in kseq group. */ - struct kseq_group *ksq_group; /* Our processor group. */ - volatile struct kse *ksq_assigned; /* assigned by another CPU. */ + LIST_ENTRY(tdq) ksq_siblings; /* Next in tdq group. */ + struct tdq_group *ksq_group; /* Our processor group. */ + volatile struct td_sched *ksq_assigned; /* assigned by another CPU. */ #else int ksq_sysload; /* For loadavg, !ITHD load. */ #endif @@ -229,21 +226,21 @@ struct kseq { #ifdef SMP /* - * kseq groups are groups of processors which can cheaply share threads. When + * tdq groups are groups of processors which can cheaply share threads. When * one processor in the group goes idle it will check the runqs of the other * processors in its group prior to halting and waiting for an interrupt. * These groups are suitable for SMT (Symetric Multi-Threading) and not NUMA. * In a numa environment we'd want an idle bitmap per group and a two tiered * load balancer. */ -struct kseq_group { - int ksg_cpus; /* Count of CPUs in this kseq group. */ +struct tdq_group { + int ksg_cpus; /* Count of CPUs in this tdq group. */ cpumask_t ksg_cpumask; /* Mask of cpus in this group. */ cpumask_t ksg_idlemask; /* Idle cpus in this group. */ cpumask_t ksg_mask; /* Bit mask for first cpu. */ int ksg_load; /* Total load of this group. */ int ksg_transferable; /* Transferable load of this group. */ - LIST_HEAD(, kseq) ksg_members; /* Linked list of all members. */ + LIST_HEAD(, tdq) ksg_members; /* Linked list of all members. */ }; #endif @@ -251,185 +248,185 @@ struct kseq_group { * One kse queue per processor. */ #ifdef SMP -static cpumask_t kseq_idle; +static cpumask_t tdq_idle; static int ksg_maxid; -static struct kseq kseq_cpu[MAXCPU]; -static struct kseq_group kseq_groups[MAXCPU]; +static struct tdq tdq_cpu[MAXCPU]; +static struct tdq_group tdq_groups[MAXCPU]; static int bal_tick; static int gbal_tick; static int balance_groups; -#define KSEQ_SELF() (&kseq_cpu[PCPU_GET(cpuid)]) -#define KSEQ_CPU(x) (&kseq_cpu[(x)]) -#define KSEQ_ID(x) ((x) - kseq_cpu) -#define KSEQ_GROUP(x) (&kseq_groups[(x)]) +#define TDQ_SELF() (&tdq_cpu[PCPU_GET(cpuid)]) +#define TDQ_CPU(x) (&tdq_cpu[(x)]) +#define TDQ_ID(x) ((x) - tdq_cpu) +#define TDQ_GROUP(x) (&tdq_groups[(x)]) #else /* !SMP */ -static struct kseq kseq_cpu; +static struct tdq tdq_cpu; -#define KSEQ_SELF() (&kseq_cpu) -#define KSEQ_CPU(x) (&kseq_cpu) +#define TDQ_SELF() (&tdq_cpu) +#define TDQ_CPU(x) (&tdq_cpu) #endif -static struct kse *sched_choose(void); /* XXX Should be thread * */ -static void sched_slice(struct kse *); +static struct td_sched *sched_choose(void); /* XXX Should be thread * */ +static void sched_slice(struct td_sched *); static void sched_priority(struct thread *); static void sched_thread_priority(struct thread *, u_char); static int sched_interact_score(struct thread *); static void sched_interact_update(struct thread *); static void sched_interact_fork(struct thread *); -static void sched_pctcpu_update(struct kse *); +static void sched_pctcpu_update(struct td_sched *); /* Operations on per processor queues */ -static struct kse * kseq_choose(struct kseq *); -static void kseq_setup(struct kseq *); -static void kseq_load_add(struct kseq *, struct kse *); -static void kseq_load_rem(struct kseq *, struct kse *); -static __inline void kseq_runq_add(struct kseq *, struct kse *, int); -static __inline void kseq_runq_rem(struct kseq *, struct kse *); -static void kseq_nice_add(struct kseq *, int); -static void kseq_nice_rem(struct kseq *, int); -void kseq_print(int cpu); +static struct td_sched * tdq_choose(struct tdq *); +static void tdq_setup(struct tdq *); +static void tdq_load_add(struct tdq *, struct td_sched *); +static void tdq_load_rem(struct tdq *, struct td_sched *); +static __inline void tdq_runq_add(struct tdq *, struct td_sched *, int); +static __inline void tdq_runq_rem(struct tdq *, struct td_sched *); +static void tdq_nice_add(struct tdq *, int); +static void tdq_nice_rem(struct tdq *, int); +void tdq_print(int cpu); #ifdef SMP -static int kseq_transfer(struct kseq *, struct kse *, int); -static struct kse *runq_steal(struct runq *); +static int tdq_transfer(struct tdq *, struct td_sched *, int); +static struct td_sched *runq_steal(struct runq *); static void sched_balance(void); static void sched_balance_groups(void); -static void sched_balance_group(struct kseq_group *); -static void sched_balance_pair(struct kseq *, struct kseq *); -static void kseq_move(struct kseq *, int); -static int kseq_idled(struct kseq *); -static void kseq_notify(struct kse *, int); -static void kseq_assign(struct kseq *); -static struct kse *kseq_steal(struct kseq *, int); -#define KSE_CAN_MIGRATE(ke) \ - ((ke)->ke_thread->td_pinned == 0 && ((ke)->ke_flags & KEF_BOUND) == 0) +static void sched_balance_group(struct tdq_group *); +static void sched_balance_pair(struct tdq *, struct tdq *); +static void tdq_move(struct tdq *, int); +static int tdq_idled(struct tdq *); +static void tdq_notify(struct td_sched *, int); +static void tdq_assign(struct tdq *); +static struct td_sched *tdq_steal(struct tdq *, int); +#define THREAD_CAN_MIGRATE(ts) \ + ((ts)->ts_thread->td_pinned == 0 && ((ts)->ts_flags & TSF_BOUND) == 0) #endif void -kseq_print(int cpu) +tdq_print(int cpu) { - struct kseq *kseq; + struct tdq *tdq; int i; - kseq = KSEQ_CPU(cpu); + tdq = TDQ_CPU(cpu); - printf("kseq:\n"); - printf("\tload: %d\n", kseq->ksq_load); - printf("\tload TIMESHARE: %d\n", kseq->ksq_load_timeshare); + printf("tdq:\n"); + printf("\tload: %d\n", tdq->ksq_load); + printf("\tload TIMESHARE: %d\n", tdq->ksq_load_timeshare); #ifdef SMP - printf("\tload transferable: %d\n", kseq->ksq_transferable); + printf("\tload transferable: %d\n", tdq->ksq_transferable); #endif - printf("\tnicemin:\t%d\n", kseq->ksq_nicemin); + printf("\tnicemin:\t%d\n", tdq->ksq_nicemin); printf("\tnice counts:\n"); for (i = 0; i < SCHED_PRI_NRESV; i++) - if (kseq->ksq_nice[i]) + if (tdq->ksq_nice[i]) printf("\t\t%d = %d\n", - i - SCHED_PRI_NHALF, kseq->ksq_nice[i]); + i - SCHED_PRI_NHALF, tdq->ksq_nice[i]); } static __inline void -kseq_runq_add(struct kseq *kseq, struct kse *ke, int flags) +tdq_runq_add(struct tdq *tdq, struct td_sched *ts, int flags) { #ifdef SMP - if (KSE_CAN_MIGRATE(ke)) { - kseq->ksq_transferable++; - kseq->ksq_group->ksg_transferable++; - ke->ke_flags |= KEF_XFERABLE; + if (THREAD_CAN_MIGRATE(ts)) { + tdq->ksq_transferable++; + tdq->ksq_group->ksg_transferable++; + ts->ts_flags |= TSF_XFERABLE; } #endif - if (ke->ke_flags & KEF_PREEMPTED) + if (ts->ts_flags & TSF_PREEMPTED) flags |= SRQ_PREEMPTED; - runq_add(ke->ke_runq, ke, flags); + runq_add(ts->ts_runq, ts, flags); } static __inline void -kseq_runq_rem(struct kseq *kseq, struct kse *ke) +tdq_runq_rem(struct tdq *tdq, struct td_sched *ts) { #ifdef SMP - if (ke->ke_flags & KEF_XFERABLE) { - kseq->ksq_transferable--; - kseq->ksq_group->ksg_transferable--; - ke->ke_flags &= ~KEF_XFERABLE; + if (ts->ts_flags & TSF_XFERABLE) { + tdq->ksq_transferable--; + tdq->ksq_group->ksg_transferable--; + ts->ts_flags &= ~TSF_XFERABLE; } #endif - runq_remove(ke->ke_runq, ke); + runq_remove(ts->ts_runq, ts); } static void -kseq_load_add(struct kseq *kseq, struct kse *ke) +tdq_load_add(struct tdq *tdq, struct td_sched *ts) { int class; mtx_assert(&sched_lock, MA_OWNED); - class = PRI_BASE(ke->ke_thread->td_pri_class); + class = PRI_BASE(ts->ts_thread->td_pri_class); if (class == PRI_TIMESHARE) - kseq->ksq_load_timeshare++; - kseq->ksq_load++; - CTR1(KTR_SCHED, "load: %d", kseq->ksq_load); - if (class != PRI_ITHD && (ke->ke_thread->td_proc->p_flag & P_NOLOAD) == 0) + tdq->ksq_load_timeshare++; + tdq->ksq_load++; + CTR1(KTR_SCHED, "load: %d", tdq->ksq_load); + if (class != PRI_ITHD && (ts->ts_thread->td_proc->p_flag & P_NOLOAD) == 0) #ifdef SMP - kseq->ksq_group->ksg_load++; + tdq->ksq_group->ksg_load++; #else - kseq->ksq_sysload++; + tdq->ksq_sysload++; #endif - if (ke->ke_thread->td_pri_class == PRI_TIMESHARE) - kseq_nice_add(kseq, ke->ke_thread->td_proc->p_nice); + if (ts->ts_thread->td_pri_class == PRI_TIMESHARE) + tdq_nice_add(tdq, ts->ts_thread->td_proc->p_nice); } static void -kseq_load_rem(struct kseq *kseq, struct kse *ke) +tdq_load_rem(struct tdq *tdq, struct td_sched *ts) { int class; mtx_assert(&sched_lock, MA_OWNED); - class = PRI_BASE(ke->ke_thread->td_pri_class); + class = PRI_BASE(ts->ts_thread->td_pri_class); if (class == PRI_TIMESHARE) - kseq->ksq_load_timeshare--; - if (class != PRI_ITHD && (ke->ke_thread->td_proc->p_flag & P_NOLOAD) == 0) + tdq->ksq_load_timeshare--; + if (class != PRI_ITHD && (ts->ts_thread->td_proc->p_flag & P_NOLOAD) == 0) #ifdef SMP - kseq->ksq_group->ksg_load--; + tdq->ksq_group->ksg_load--; #else - kseq->ksq_sysload--; + tdq->ksq_sysload--; #endif - kseq->ksq_load--; - CTR1(KTR_SCHED, "load: %d", kseq->ksq_load); - ke->ke_runq = NULL; - if (ke->ke_thread->td_pri_class == PRI_TIMESHARE) - kseq_nice_rem(kseq, ke->ke_thread->td_proc->p_nice); + tdq->ksq_load--; + CTR1(KTR_SCHED, "load: %d", tdq->ksq_load); + ts->ts_runq = NULL; + if (ts->ts_thread->td_pri_class == PRI_TIMESHARE) + tdq_nice_rem(tdq, ts->ts_thread->td_proc->p_nice); } static void -kseq_nice_add(struct kseq *kseq, int nice) +tdq_nice_add(struct tdq *tdq, int nice) { mtx_assert(&sched_lock, MA_OWNED); /* Normalize to zero. */ - kseq->ksq_nice[nice + SCHED_PRI_NHALF]++; - if (nice < kseq->ksq_nicemin || kseq->ksq_load_timeshare == 1) - kseq->ksq_nicemin = nice; + tdq->ksq_nice[nice + SCHED_PRI_NHALF]++; + if (nice < tdq->ksq_nicemin || tdq->ksq_load_timeshare == 1) + tdq->ksq_nicemin = nice; } static void -kseq_nice_rem(struct kseq *kseq, int nice) +tdq_nice_rem(struct tdq *tdq, int nice) { int n; mtx_assert(&sched_lock, MA_OWNED); /* Normalize to zero. */ n = nice + SCHED_PRI_NHALF; - kseq->ksq_nice[n]--; - KASSERT(kseq->ksq_nice[n] >= 0, ("Negative nice count.")); + tdq->ksq_nice[n]--; + KASSERT(tdq->ksq_nice[n] >= 0, ("Negative nice count.")); /* * If this wasn't the smallest nice value or there are more in * this bucket we can just return. Otherwise we have to recalculate * the smallest nice. */ - if (nice != kseq->ksq_nicemin || - kseq->ksq_nice[n] != 0 || - kseq->ksq_load_timeshare == 0) + if (nice != tdq->ksq_nicemin || + tdq->ksq_nice[n] != 0 || + tdq->ksq_load_timeshare == 0) return; for (; n < SCHED_PRI_NRESV; n++) - if (kseq->ksq_nice[n]) { - kseq->ksq_nicemin = n - SCHED_PRI_NHALF; + if (tdq->ksq_nice[n]) { + tdq->ksq_nicemin = n - SCHED_PRI_NHALF; return; } } @@ -454,9 +451,9 @@ kseq_nice_rem(struct kseq *kseq, int nice) static void sched_balance(void) { - struct kseq_group *high; - struct kseq_group *low; - struct kseq_group *ksg; + struct tdq_group *high; + struct tdq_group *low; + struct tdq_group *ksg; int cnt; int i; @@ -466,7 +463,7 @@ sched_balance(void) low = high = NULL; i = random() % (ksg_maxid + 1); for (cnt = 0; cnt <= ksg_maxid; cnt++) { - ksg = KSEQ_GROUP(i); + ksg = TDQ_GROUP(i); /* * Find the CPU with the highest load that has some * threads to transfer. @@ -493,34 +490,34 @@ sched_balance_groups(void) mtx_assert(&sched_lock, MA_OWNED); if (smp_started) for (i = 0; i <= ksg_maxid; i++) - sched_balance_group(KSEQ_GROUP(i)); + sched_balance_group(TDQ_GROUP(i)); } static void -sched_balance_group(struct kseq_group *ksg) +sched_balance_group(struct tdq_group *ksg) { - struct kseq *kseq; - struct kseq *high; - struct kseq *low; + struct tdq *tdq; + struct tdq *high; + struct tdq *low; int load; if (ksg->ksg_transferable == 0) return; low = NULL; high = NULL; - LIST_FOREACH(kseq, &ksg->ksg_members, ksq_siblings) { - load = kseq->ksq_load; + LIST_FOREACH(tdq, &ksg->ksg_members, ksq_siblings) { + load = tdq->ksq_load; if (high == NULL || load > high->ksq_load) - high = kseq; + high = tdq; if (low == NULL || load < low->ksq_load) - low = kseq; + low = tdq; } if (high != NULL && low != NULL && high != low) sched_balance_pair(high, low); } static void -sched_balance_pair(struct kseq *high, struct kseq *low) +sched_balance_pair(struct tdq *high, struct tdq *low) { int transferable; int high_load; @@ -531,7 +528,7 @@ sched_balance_pair(struct kseq *high, struct kseq *low) /* * If we're transfering within a group we have to use this specific - * kseq's transferable count, otherwise we can steal from other members + * tdq's transferable count, otherwise we can steal from other members * of the group. */ if (high->ksq_group == low->ksq_group) { @@ -555,135 +552,135 @@ sched_balance_pair(struct kseq *high, struct kseq *low) move++; move = min(move, transferable); for (i = 0; i < move; i++) - kseq_move(high, KSEQ_ID(low)); + tdq_move(high, TDQ_ID(low)); return; } static void -kseq_move(struct kseq *from, int cpu) +tdq_move(struct tdq *from, int cpu) { - struct kseq *kseq; - struct kseq *to; - struct kse *ke; + struct tdq *tdq; + struct tdq *to; + struct td_sched *ts; - kseq = from; - to = KSEQ_CPU(cpu); - ke = kseq_steal(kseq, 1); - if (ke == NULL) { - struct kseq_group *ksg; + tdq = from; + to = TDQ_CPU(cpu); + ts = tdq_steal(tdq, 1); + if (ts == NULL) { + struct tdq_group *ksg; - ksg = kseq->ksq_group; - LIST_FOREACH(kseq, &ksg->ksg_members, ksq_siblings) { - if (kseq == from || kseq->ksq_transferable == 0) + ksg = tdq->ksq_group; + LIST_FOREACH(tdq, &ksg->ksg_members, ksq_siblings) { + if (tdq == from || tdq->ksq_transferable == 0) continue; - ke = kseq_steal(kseq, 1); + ts = tdq_steal(tdq, 1); break; } - if (ke == NULL) - panic("kseq_move: No KSEs available with a " + if (ts == NULL) + panic("tdq_move: No threads available with a " "transferable count of %d\n", ksg->ksg_transferable); } - if (kseq == to) + if (tdq == to) return; - ke->ke_state = KES_THREAD; - kseq_runq_rem(kseq, ke); - kseq_load_rem(kseq, ke); - kseq_notify(ke, cpu); + ts->ts_state = TSS_THREAD; + tdq_runq_rem(tdq, ts); + tdq_load_rem(tdq, ts); + tdq_notify(ts, cpu); } static int -kseq_idled(struct kseq *kseq) +tdq_idled(struct tdq *tdq) { - struct kseq_group *ksg; - struct kseq *steal; - struct kse *ke; + struct tdq_group *ksg; + struct tdq *steal; + struct td_sched *ts; - ksg = kseq->ksq_group; + ksg = tdq->ksq_group; /* * If we're in a cpu group, try and steal kses from another cpu in * the group before idling. */ if (ksg->ksg_cpus > 1 && ksg->ksg_transferable) { LIST_FOREACH(steal, &ksg->ksg_members, ksq_siblings) { - if (steal == kseq || steal->ksq_transferable == 0) + if (steal == tdq || steal->ksq_transferable == 0) continue; - ke = kseq_steal(steal, 0); - if (ke == NULL) + ts = tdq_steal(steal, 0); + if (ts == NULL) continue; - ke->ke_state = KES_THREAD; - kseq_runq_rem(steal, ke); - kseq_load_rem(steal, ke); - ke->ke_cpu = PCPU_GET(cpuid); - ke->ke_flags |= KEF_INTERNAL | KEF_HOLD; - sched_add(ke->ke_thread, SRQ_YIELDING); + ts->ts_state = TSS_THREAD; + tdq_runq_rem(steal, ts); + tdq_load_rem(steal, ts); + ts->ts_cpu = PCPU_GET(cpuid); + ts->ts_flags |= TSF_INTERNAL | TSF_HOLD; + sched_add(ts->ts_thread, SRQ_YIELDING); return (0); } } /* * We only set the idled bit when all of the cpus in the group are - * idle. Otherwise we could get into a situation where a KSE bounces + * idle. Otherwise we could get into a situation where a thread bounces * back and forth between two idle cores on seperate physical CPUs. */ ksg->ksg_idlemask |= PCPU_GET(cpumask); if (ksg->ksg_idlemask != ksg->ksg_cpumask) return (1); - atomic_set_int(&kseq_idle, ksg->ksg_mask); + atomic_set_int(&tdq_idle, ksg->ksg_mask); return (1); } static void -kseq_assign(struct kseq *kseq) +tdq_assign(struct tdq *tdq) { - struct kse *nke; - struct kse *ke; + struct td_sched *nts; + struct td_sched *ts; do { - *(volatile struct kse **)&ke = kseq->ksq_assigned; - } while(!atomic_cmpset_ptr((volatile uintptr_t *)&kseq->ksq_assigned, - (uintptr_t)ke, (uintptr_t)NULL)); - for (; ke != NULL; ke = nke) { - nke = ke->ke_assign; - kseq->ksq_group->ksg_load--; - kseq->ksq_load--; - ke->ke_flags &= ~KEF_ASSIGNED; - if (ke->ke_flags & KEF_REMOVED) { - ke->ke_flags &= ~KEF_REMOVED; + *(volatile struct td_sched **)&ts = tdq->ksq_assigned; + } while(!atomic_cmpset_ptr((volatile uintptr_t *)&tdq->ksq_assigned, + (uintptr_t)ts, (uintptr_t)NULL)); + for (; ts != NULL; ts = nts) { + nts = ts->ts_assign; + tdq->ksq_group->ksg_load--; + tdq->ksq_load--; + ts->ts_flags &= ~TSF_ASSIGNED; + if (ts->ts_flags & TSF_REMOVED) { + ts->ts_flags &= ~TSF_REMOVED; continue; } - ke->ke_flags |= KEF_INTERNAL | KEF_HOLD; - sched_add(ke->ke_thread, SRQ_YIELDING); + ts->ts_flags |= TSF_INTERNAL | TSF_HOLD; + sched_add(ts->ts_thread, SRQ_YIELDING); } } static void -kseq_notify(struct kse *ke, int cpu) +tdq_notify(struct td_sched *ts, int cpu) { - struct kseq *kseq; + struct tdq *tdq; struct thread *td; struct pcpu *pcpu; int class; int prio; - kseq = KSEQ_CPU(cpu); + tdq = TDQ_CPU(cpu); /* XXX */ - class = PRI_BASE(ke->ke_thread->td_pri_class); + class = PRI_BASE(ts->ts_thread->td_pri_class); if ((class == PRI_TIMESHARE || class == PRI_REALTIME) && - (kseq_idle & kseq->ksq_group->ksg_mask)) - atomic_clear_int(&kseq_idle, kseq->ksq_group->ksg_mask); - kseq->ksq_group->ksg_load++; - kseq->ksq_load++; - ke->ke_cpu = cpu; - ke->ke_flags |= KEF_ASSIGNED; - prio = ke->ke_thread->td_priority; + (tdq_idle & tdq->ksq_group->ksg_mask)) + atomic_clear_int(&tdq_idle, tdq->ksq_group->ksg_mask); + tdq->ksq_group->ksg_load++; + tdq->ksq_load++; + ts->ts_cpu = cpu; + ts->ts_flags |= TSF_ASSIGNED; + prio = ts->ts_thread->td_priority; /* - * Place a KSE on another cpu's queue and force a resched. + * Place a thread on another cpu's queue and force a resched. */ do { - *(volatile struct kse **)&ke->ke_assign = kseq->ksq_assigned; - } while(!atomic_cmpset_ptr((volatile uintptr_t *)&kseq->ksq_assigned, - (uintptr_t)ke->ke_assign, (uintptr_t)ke)); + *(volatile struct td_sched **)&ts->ts_assign = tdq->ksq_assigned; + } while(!atomic_cmpset_ptr((volatile uintptr_t *)&tdq->ksq_assigned, + (uintptr_t)ts->ts_assign, (uintptr_t)ts)); /* * Without sched_lock we could lose a race where we set NEEDRESCHED * on a thread that is switched out before the IPI is delivered. This @@ -692,19 +689,19 @@ kseq_notify(struct kse *ke, int cpu) */ pcpu = pcpu_find(cpu); td = pcpu->pc_curthread; - if (ke->ke_thread->td_priority < td->td_priority || + if (ts->ts_thread->td_priority < td->td_priority || td == pcpu->pc_idlethread) { td->td_flags |= TDF_NEEDRESCHED; ipi_selected(1 << cpu, IPI_AST); } } -static struct kse * +static struct td_sched * runq_steal(struct runq *rq) { struct rqhead *rqh; struct rqbits *rqb; - struct kse *ke; + struct td_sched *ts; int word; int bit; @@ -717,39 +714,39 @@ runq_steal(struct runq *rq) if ((rqb->rqb_bits[word] & (1ul << bit)) == 0) continue; rqh = &rq->rq_queues[bit + (word << RQB_L2BPW)]; - TAILQ_FOREACH(ke, rqh, ke_procq) { - if (KSE_CAN_MIGRATE(ke)) - return (ke); + TAILQ_FOREACH(ts, rqh, ts_procq) { + if (THREAD_CAN_MIGRATE(ts)) + return (ts); } } } return (NULL); } -static struct kse * -kseq_steal(struct kseq *kseq, int stealidle) +static struct td_sched * +tdq_steal(struct tdq *tdq, int stealidle) { - struct kse *ke; + struct td_sched *ts; /* * Steal from next first to try to get a non-interactive task that * may not have run for a while. */ - if ((ke = runq_steal(kseq->ksq_next)) != NULL) - return (ke); - if ((ke = runq_steal(kseq->ksq_curr)) != NULL) - return (ke); + if ((ts = runq_steal(tdq->ksq_next)) != NULL) + return (ts); + if ((ts = runq_steal(tdq->ksq_curr)) != NULL) + return (ts); if (stealidle) - return (runq_steal(&kseq->ksq_idle)); + return (runq_steal(&tdq->ksq_idle)); return (NULL); } int -kseq_transfer(struct kseq *kseq, struct kse *ke, int class) +tdq_transfer(struct tdq *tdq, struct td_sched *ts, int class) { - struct kseq_group *nksg; - struct kseq_group *ksg; - struct kseq *old; + struct tdq_group *nksg; + struct tdq_group *ksg; + struct tdq *old; int cpu; int idx; @@ -767,16 +764,16 @@ kseq_transfer(struct kseq *kseq, struct kse *ke, int class) * some CPUs may idle. Too low and there will be excess migration * and context switches. */ - old = KSEQ_CPU(ke->ke_cpu); + old = TDQ_CPU(ts->ts_cpu); nksg = old->ksq_group; - ksg = kseq->ksq_group; - if (kseq_idle) { - if (kseq_idle & nksg->ksg_mask) { + ksg = tdq->ksq_group; + if (tdq_idle) { + if (tdq_idle & nksg->ksg_mask) { cpu = ffs(nksg->ksg_idlemask); if (cpu) { CTR2(KTR_SCHED, - "kseq_transfer: %p found old cpu %X " - "in idlemask.", ke, cpu); + "tdq_transfer: %p found old cpu %X " + "in idlemask.", ts, cpu); goto migrate; } } @@ -784,30 +781,30 @@ kseq_transfer(struct kseq *kseq, struct kse *ke, int class) * Multiple cpus could find this bit simultaneously * but the race shouldn't be terrible. */ - cpu = ffs(kseq_idle); + cpu = ffs(tdq_idle); if (cpu) { - CTR2(KTR_SCHED, "kseq_transfer: %p found %X " - "in idlemask.", ke, cpu); + CTR2(KTR_SCHED, "tdq_transfer: %p found %X " + "in idlemask.", ts, cpu); goto migrate; } } idx = 0; #if 0 - if (old->ksq_load < kseq->ksq_load) { - cpu = ke->ke_cpu + 1; - CTR2(KTR_SCHED, "kseq_transfer: %p old cpu %X " - "load less than ours.", ke, cpu); + if (old->ksq_load < tdq->ksq_load) { + cpu = ts->ts_cpu + 1; + CTR2(KTR_SCHED, "tdq_transfer: %p old cpu %X " + "load less than ours.", ts, cpu); goto migrate; } /* * No new CPU was found, look for one with less load. */ for (idx = 0; idx <= ksg_maxid; idx++) { - nksg = KSEQ_GROUP(idx); + nksg = TDQ_GROUP(idx); if (nksg->ksg_load /*+ (nksg->ksg_cpus * 2)*/ < ksg->ksg_load) { cpu = ffs(nksg->ksg_cpumask); - CTR2(KTR_SCHED, "kseq_transfer: %p cpu %X load less " - "than ours.", ke, cpu); + CTR2(KTR_SCHED, "tdq_transfer: %p cpu %X load less " + "than ours.", ts, cpu); goto migrate; } } @@ -819,8 +816,8 @@ kseq_transfer(struct kseq *kseq, struct kse *ke, int class) if (ksg->ksg_idlemask) { cpu = ffs(ksg->ksg_idlemask); if (cpu) { - CTR2(KTR_SCHED, "kseq_transfer: %p cpu %X idle in " - "group.", ke, cpu); + CTR2(KTR_SCHED, "tdq_transfer: %p cpu %X idle in " + "group.", ts, cpu); goto migrate; } } @@ -830,8 +827,8 @@ kseq_transfer(struct kseq *kseq, struct kse *ke, int class) * Now that we've found an idle CPU, migrate the thread. */ cpu--; - ke->ke_runq = NULL; - kseq_notify(ke, cpu); + ts->ts_runq = NULL; + tdq_notify(ts, cpu); return (1); } @@ -842,61 +839,61 @@ kseq_transfer(struct kseq *kseq, struct kse *ke, int class) * Pick the highest priority task we have and return it. */ -static struct kse * -kseq_choose(struct kseq *kseq) +static struct td_sched * +tdq_choose(struct tdq *tdq) { struct runq *swap; - struct kse *ke; + struct td_sched *ts; int nice; mtx_assert(&sched_lock, MA_OWNED); swap = NULL; for (;;) { - ke = runq_choose(kseq->ksq_curr); - if (ke == NULL) { + ts = runq_choose(tdq->ksq_curr); + if (ts == NULL) { /* * We already swapped once and didn't get anywhere. */ if (swap) break; - swap = kseq->ksq_curr; - kseq->ksq_curr = kseq->ksq_next; - kseq->ksq_next = swap; + swap = tdq->ksq_curr; + tdq->ksq_curr = tdq->ksq_next; + tdq->ksq_next = swap; continue; } /* - * If we encounter a slice of 0 the kse is in a - * TIMESHARE kse group and its nice was too far out + * If we encounter a slice of 0 the td_sched is in a + * TIMESHARE td_sched group and its nice was too far out * of the range that receives slices. */ - nice = ke->ke_thread->td_proc->p_nice + (0 - kseq->ksq_nicemin); + nice = ts->ts_thread->td_proc->p_nice + (0 - tdq->ksq_nicemin); #if 0 - if (ke->ke_slice == 0 || (nice > SCHED_SLICE_NTHRESH && - ke->ke_thread->td_proc->p_nice != 0)) { - runq_remove(ke->ke_runq, ke); - sched_slice(ke); - ke->ke_runq = kseq->ksq_next; - runq_add(ke->ke_runq, ke, 0); + if (ts->ts_slice == 0 || (nice > SCHED_SLICE_NTHRESH && + ts->ts_thread->td_proc->p_nice != 0)) { + runq_remove(ts->ts_runq, ts); + sched_slice(ts); + ts->ts_runq = tdq->ksq_next; + runq_add(ts->ts_runq, ts, 0); continue; } #endif - return (ke); + return (ts); } - return (runq_choose(&kseq->ksq_idle)); + return (runq_choose(&tdq->ksq_idle)); } static void -kseq_setup(struct kseq *kseq) +tdq_setup(struct tdq *tdq) { - runq_init(&kseq->ksq_timeshare[0]); - runq_init(&kseq->ksq_timeshare[1]); - runq_init(&kseq->ksq_idle); - kseq->ksq_curr = &kseq->ksq_timeshare[0]; - kseq->ksq_next = &kseq->ksq_timeshare[1]; - kseq->ksq_load = 0; - kseq->ksq_load_timeshare = 0; + runq_init(&tdq->ksq_timeshare[0]); + runq_init(&tdq->ksq_timeshare[1]); + runq_init(&tdq->ksq_idle); + tdq->ksq_curr = &tdq->ksq_timeshare[0]; + tdq->ksq_next = &tdq->ksq_timeshare[1]; + tdq->ksq_load = 0; + tdq->ksq_load_timeshare = 0; } static void @@ -917,27 +914,27 @@ sched_setup(void *dummy) #ifdef SMP balance_groups = 0; /* - * Initialize the kseqs. + * Initialize the tdqs. */ for (i = 0; i < MAXCPU; i++) { - struct kseq *ksq; + struct tdq *ksq; - ksq = &kseq_cpu[i]; + ksq = &tdq_cpu[i]; ksq->ksq_assigned = NULL; - kseq_setup(&kseq_cpu[i]); + tdq_setup(&tdq_cpu[i]); } if (smp_topology == NULL) { - struct kseq_group *ksg; - struct kseq *ksq; + struct tdq_group *ksg; + struct tdq *ksq; int cpus; for (cpus = 0, i = 0; i < MAXCPU; i++) { if (CPU_ABSENT(i)) continue; - ksq = &kseq_cpu[i]; - ksg = &kseq_groups[cpus]; + ksq = &tdq_cpu[i]; + ksg = &tdq_groups[cpus]; /* - * Setup a kseq group with one member. + * Setup a tdq group with one member. */ ksq->ksq_transferable = 0; ksq->ksq_group = ksg; @@ -952,13 +949,13 @@ sched_setup(void *dummy) } ksg_maxid = cpus - 1; } else { - struct kseq_group *ksg; + struct tdq_group *ksg; struct cpu_group *cg; int j; for (i = 0; i < smp_topology->ct_count; i++) { cg = &smp_topology->ct_group[i]; - ksg = &kseq_groups[i]; + ksg = &tdq_groups[i]; /* * Initialize the group. */ @@ -975,10 +972,10 @@ sched_setup(void *dummy) if ((cg->cg_mask & (1 << j)) != 0) { if (ksg->ksg_mask == 0) ksg->ksg_mask = 1 << j; - kseq_cpu[j].ksq_transferable = 0; - kseq_cpu[j].ksq_group = ksg; + tdq_cpu[j].ksq_transferable = 0; + tdq_cpu[j].ksq_group = ksg; LIST_INSERT_HEAD(&ksg->ksg_members, - &kseq_cpu[j], ksq_siblings); + &tdq_cpu[j], ksq_siblings); } } if (ksg->ksg_cpus > 1) @@ -994,10 +991,10 @@ sched_setup(void *dummy) if (balance_groups) gbal_tick = ticks + (hz / 2); #else - kseq_setup(KSEQ_SELF()); + tdq_setup(TDQ_SELF()); #endif mtx_lock_spin(&sched_lock); - kseq_load_add(KSEQ_SELF(), &kse0); + tdq_load_add(TDQ_SELF(), &td_sched0); mtx_unlock_spin(&sched_lock); } @@ -1042,71 +1039,67 @@ sched_priority(struct thread *td) else if (pri < PRI_MIN_TIMESHARE) pri = PRI_MIN_TIMESHARE; -#ifdef KSE - sched_user_prio(kg, pri); -#else sched_user_prio(td, pri); -#endif return; } /* - * Calculate a time slice based on the properties of the kseg and the runq - * that we're on. This is only for PRI_TIMESHARE threads. + * Calculate a time slice based on the properties of the process + * and the runq that we're on. This is only for PRI_TIMESHARE threads. */ static void -sched_slice(struct kse *ke) +sched_slice(struct td_sched *ts) { - struct kseq *kseq; + struct tdq *tdq; struct thread *td; - td = ke->ke_thread; - kseq = KSEQ_CPU(ke->ke_cpu); + td = ts->ts_thread; + tdq = TDQ_CPU(ts->ts_cpu); if (td->td_flags & TDF_BORROWING) { - ke->ke_slice = SCHED_SLICE_MIN; + ts->ts_slice = SCHED_SLICE_MIN; return; } /* * Rationale: - * KSEs in interactive ksegs get a minimal slice so that we + * Threads in interactive procs get a minimal slice so that we * quickly notice if it abuses its advantage. * - * KSEs in non-interactive ksegs are assigned a slice that is - * based on the ksegs nice value relative to the least nice kseg + * Threads in non-interactive procs are assigned a slice that is + * based on the procs nice value relative to the least nice procs * on the run queue for this cpu. * - * If the KSE is less nice than all others it gets the maximum - * slice and other KSEs will adjust their slice relative to + * If the thread is less nice than all others it gets the maximum + * slice and other threads will adjust their slice relative to * this when they first expire. * * There is 20 point window that starts relative to the least - * nice kse on the run queue. Slice size is determined by - * the kse distance from the last nice thread. + * nice td_sched on the run queue. Slice size is determined by + * the td_sched distance from the last nice thread. * - * If the kse is outside of the window it will get no slice + * If the td_sched is outside of the window it will get no slice * and will be reevaluated each time it is selected on the - * run queue. The exception to this is nice 0 ksegs when + * run queue. The exception to this is nice 0 procs when * a nice -20 is running. They are always granted a minimum * slice. */ if (!SCHED_INTERACTIVE(td)) { int nice; - nice = td->td_proc->p_nice + (0 - kseq->ksq_nicemin); - if (kseq->ksq_load_timeshare == 0 || - td->td_proc->p_nice < kseq->ksq_nicemin) - ke->ke_slice = SCHED_SLICE_MAX; + nice = td->td_proc->p_nice + (0 - tdq->ksq_nicemin); + if (tdq->ksq_load_timeshare == 0 || + td->td_proc->p_nice < tdq->ksq_nicemin) + ts->ts_slice = SCHED_SLICE_MAX; else if (nice <= SCHED_SLICE_NTHRESH) - ke->ke_slice = SCHED_SLICE_NICE(nice); + ts->ts_slice = SCHED_SLICE_NICE(nice); else if (td->td_proc->p_nice == 0) - ke->ke_slice = SCHED_SLICE_MIN; + ts->ts_slice = SCHED_SLICE_MIN; else - ke->ke_slice = SCHED_SLICE_MIN; /* 0 */ + ts->ts_slice = SCHED_SLICE_MIN; /* 0 */ } else - ke->ke_slice = SCHED_SLICE_INTERACTIVE; + ts->ts_slice = SCHED_SLICE_INTERACTIVE; return; } @@ -1187,9 +1180,9 @@ schedinit(void) * Set up the scheduler specific parts of proc0. */ proc0.p_sched = NULL; /* XXX */ - thread0.td_sched = &kse0; - kse0.ke_thread = &thread0; - kse0.ke_state = KES_THREAD; + thread0.td_sched = &td_sched0; + td_sched0.ts_thread = &thread0; + td_sched0.ts_state = TSS_THREAD; } /* @@ -1204,35 +1197,35 @@ sched_rr_interval(void) } static void -sched_pctcpu_update(struct kse *ke) +sched_pctcpu_update(struct td_sched *ts) { /* * Adjust counters and watermark for pctcpu calc. */ - if (ke->ke_ltick > ticks - SCHED_CPU_TICKS) { + if (ts->ts_ltick > ticks - SCHED_CPU_TICKS) { /* * Shift the tick count out so that the divide doesn't * round away our results. */ - ke->ke_ticks <<= 10; - ke->ke_ticks = (ke->ke_ticks / (ticks - ke->ke_ftick)) * + ts->ts_ticks <<= 10; + ts->ts_ticks = (ts->ts_ticks / (ticks - ts->ts_ftick)) * SCHED_CPU_TICKS; - ke->ke_ticks >>= 10; + ts->ts_ticks >>= 10; } else - ke->ke_ticks = 0; - ke->ke_ltick = ticks; - ke->ke_ftick = ke->ke_ltick - SCHED_CPU_TICKS; + ts->ts_ticks = 0; + ts->ts_ltick = ticks; + ts->ts_ftick = ts->ts_ltick - SCHED_CPU_TICKS; } void sched_thread_priority(struct thread *td, u_char prio) { - struct kse *ke; + struct td_sched *ts; CTR6(KTR_SCHED, "sched_prio: %p(%s) prio %d newprio %d by %p(%s)", td, td->td_proc->p_comm, td->td_priority, prio, curthread, curthread->td_proc->p_comm); - ke = td->td_kse; + ts = td->td_sched; mtx_assert(&sched_lock, MA_OWNED); if (td->td_priority == prio) return; @@ -1243,21 +1236,21 @@ sched_thread_priority(struct thread *td, u_char prio) * queue. We still call adjustrunqueue below in case kse * needs to fix things up. */ - if (prio < td->td_priority && ke->ke_runq != NULL && - (ke->ke_flags & KEF_ASSIGNED) == 0 && - ke->ke_runq != KSEQ_CPU(ke->ke_cpu)->ksq_curr) { - runq_remove(ke->ke_runq, ke); - ke->ke_runq = KSEQ_CPU(ke->ke_cpu)->ksq_curr; - runq_add(ke->ke_runq, ke, 0); + if (prio < td->td_priority && ts->ts_runq != NULL && + (ts->ts_flags & TSF_ASSIGNED) == 0 && + ts->ts_runq != TDQ_CPU(ts->ts_cpu)->ksq_curr) { + runq_remove(ts->ts_runq, ts); + ts->ts_runq = TDQ_CPU(ts->ts_cpu)->ksq_curr; + runq_add(ts->ts_runq, ts, 0); } /* - * Hold this kse on this cpu so that sched_prio() doesn't + * Hold this td_sched on this cpu so that sched_prio() doesn't * cause excessive migration. We only want migration to * happen as the result of a wakeup. */ - ke->ke_flags |= KEF_HOLD; + ts->ts_flags |= TSF_HOLD; adjustrunqueue(td, prio); - ke->ke_flags &= ~KEF_HOLD; + ts->ts_flags &= ~TSF_HOLD; } else td->td_priority = prio; } @@ -1327,39 +1320,14 @@ sched_prio(struct thread *td, u_char prio) } void -#ifdef KSE -sched_user_prio(struct ksegrp *kg, u_char prio) -#else sched_user_prio(struct thread *td, u_char prio) -#endif { -#ifdef KSE - struct thread *td; -#endif u_char oldprio; -#ifdef KSE - kg->kg_base_user_pri = prio; - - /* XXXKSE only for 1:1 */ - - td = TAILQ_FIRST(&kg->kg_threads); - if (td == NULL) { - kg->kg_user_pri = prio; - return; - } - - if (td->td_flags & TDF_UBORROWING && kg->kg_user_pri <= prio) - return; - - oldprio = kg->kg_user_pri; - kg->kg_user_pri = prio; -#else td->td_base_user_pri = prio; oldprio = td->td_user_pri; td->td_user_pri = prio; -#endif if (TD_ON_UPILOCK(td) && oldprio != prio) umtx_pi_adjust(td, oldprio); @@ -1372,13 +1340,8 @@ sched_lend_user_prio(struct thread *td, u_char prio) td->td_flags |= TDF_UBORROWING; -#ifdef KSE - oldprio = td->td_ksegrp->kg_user_pri; - td->td_ksegrp->kg_user_pri = prio; -#else oldprio = td->td_user_pri; td->td_user_pri = prio; -#endif if (TD_ON_UPILOCK(td) && oldprio != prio) umtx_pi_adjust(td, oldprio); @@ -1387,23 +1350,12 @@ sched_lend_user_prio(struct thread *td, u_char prio) void sched_unlend_user_prio(struct thread *td, u_char prio) { -#ifdef KSE - struct ksegrp *kg = td->td_ksegrp; -#endif u_char base_pri; -#ifdef KSE - base_pri = kg->kg_base_user_pri; -#else base_pri = td->td_base_user_pri; -#endif if (prio >= base_pri) { td->td_flags &= ~TDF_UBORROWING; -#ifdef KSE - sched_user_prio(kg, base_pri); -#else sched_user_prio(td, base_pri); -#endif } else sched_lend_user_prio(td, prio); } @@ -1411,13 +1363,13 @@ sched_unlend_user_prio(struct thread *td, u_char prio) void sched_switch(struct thread *td, struct thread *newtd, int flags) { - struct kseq *ksq; - struct kse *ke; + struct tdq *ksq; + struct td_sched *ts; mtx_assert(&sched_lock, MA_OWNED); - ke = td->td_kse; - ksq = KSEQ_SELF(); + ts = td->td_sched; + ksq = TDQ_SELF(); td->td_lastcpu = td->td_oncpu; td->td_oncpu = NOCPU; @@ -1425,24 +1377,24 @@ sched_switch(struct thread *td, struct thread *newtd, int flags) td->td_owepreempt = 0; /* - * If the KSE has been assigned it may be in the process of switching + * If the thread has been assigned it may be in the process of switching * to the new cpu. This is the case in sched_bind(). */ if (td == PCPU_GET(idlethread)) { TD_SET_CAN_RUN(td); - } else if ((ke->ke_flags & KEF_ASSIGNED) == 0) { + } else if ((ts->ts_flags & TSF_ASSIGNED) == 0) { /* We are ending our run so make our slot available again */ - kseq_load_rem(ksq, ke); + tdq_load_rem(ksq, ts); if (TD_IS_RUNNING(td)) { /* * Don't allow the thread to migrate * from a preemption. */ - ke->ke_flags |= KEF_HOLD; + ts->ts_flags |= TSF_HOLD; setrunqueue(td, (flags & SW_PREEMPT) ? SRQ_OURSELF|SRQ_YIELDING|SRQ_PREEMPTED : SRQ_OURSELF|SRQ_YIELDING); - ke->ke_flags &= ~KEF_HOLD; + ts->ts_flags &= ~TSF_HOLD; } } if (newtd != NULL) { @@ -1450,10 +1402,10 @@ sched_switch(struct thread *td, struct thread *newtd, int flags) * If we bring in a thread account for it as if it had been * added to the run queue and then chosen. */ - newtd->td_kse->ke_flags |= KEF_DIDRUN; - newtd->td_kse->ke_runq = ksq->ksq_curr; + newtd->td_sched->ts_flags |= TSF_DIDRUN; + newtd->td_sched->ts_runq = ksq->ksq_curr; TD_SET_RUNNING(newtd); - kseq_load_add(KSEQ_SELF(), newtd->td_kse); + tdq_load_add(TDQ_SELF(), newtd->td_sched); } else newtd = choosethread(); if (td != newtd) { @@ -1477,23 +1429,23 @@ sched_switch(struct thread *td, struct thread *newtd, int flags) void sched_nice(struct proc *p, int nice) { - struct kse *ke; + struct td_sched *ts; struct thread *td; - struct kseq *kseq; + struct tdq *tdq; PROC_LOCK_ASSERT(p, MA_OWNED); mtx_assert(&sched_lock, MA_OWNED); /* - * We need to adjust the nice counts for running KSEs. + * We need to adjust the nice counts for running threads. */ FOREACH_THREAD_IN_PROC(p, td) { if (td->td_pri_class == PRI_TIMESHARE) { - ke = td->td_kse; - if (ke->ke_runq == NULL) + ts = td->td_sched; + if (ts->ts_runq == NULL) continue; - kseq = KSEQ_CPU(ke->ke_cpu); - kseq_nice_rem(kseq, p->p_nice); - kseq_nice_add(kseq, nice); + tdq = TDQ_CPU(ts->ts_cpu); + tdq_nice_rem(tdq, p->p_nice); + tdq_nice_add(tdq, nice); } } p->p_nice = nice; @@ -1508,7 +1460,7 @@ sched_sleep(struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); - td->td_kse->ke_slptime = ticks; + td->td_sched->ts_slptime = ticks; } void @@ -1517,13 +1469,13 @@ sched_wakeup(struct thread *td) mtx_assert(&sched_lock, MA_OWNED); /* - * Let the kseg know how long we slept for. This is because process - * interactivity behavior is modeled in the kseg. + * Let the procs know how long we slept for. This is because process + * interactivity behavior is modeled in the procs. */ - if (td->td_kse->ke_slptime) { + if (td->td_sched->ts_slptime) { int hzticks; - hzticks = (ticks - td->td_kse->ke_slptime) << 10; + hzticks = (ticks - td->td_sched->ts_slptime) << 10; if (hzticks >= SCHED_SLP_RUN_MAX) { td->td_sched->skg_slptime = SCHED_SLP_RUN_MAX; td->td_sched->skg_runtime = 1; @@ -1532,8 +1484,8 @@ sched_wakeup(struct thread *td) sched_interact_update(td); } sched_priority(td); - sched_slice(td->td_kse); - td->td_kse->ke_slptime = 0; + sched_slice(td->td_sched); + td->td_sched->ts_slptime = 0; } setrunqueue(td, SRQ_BORING); } @@ -1545,10 +1497,15 @@ sched_wakeup(struct thread *td) void sched_fork(struct thread *td, struct thread *child) { - struct kse *ke; - struct kse *ke2; - mtx_assert(&sched_lock, MA_OWNED); + sched_fork_thread(td, child); +} + +void +sched_fork_thread(struct thread *td, struct thread *child) +{ + struct td_sched *ts; + struct td_sched *ts2; child->td_sched->skg_slptime = td->td_sched->skg_slptime; child->td_sched->skg_runtime = td->td_sched->skg_runtime; @@ -1560,23 +1517,23 @@ sched_fork(struct thread *td, struct thread *child) sched_newthread(child); - ke = td->td_kse; - ke2 = child->td_kse; - ke2->ke_slice = 1; /* Attempt to quickly learn interactivity. */ - ke2->ke_cpu = ke->ke_cpu; - ke2->ke_runq = NULL; + ts = td->td_sched; + ts2 = child->td_sched; + ts2->ts_slice = 1; /* Attempt to quickly learn interactivity. */ + ts2->ts_cpu = ts->ts_cpu; + ts2->ts_runq = NULL; /* Grab our parents cpu estimation information. */ - ke2->ke_ticks = ke->ke_ticks; - ke2->ke_ltick = ke->ke_ltick; - ke2->ke_ftick = ke->ke_ftick; + ts2->ts_ticks = ts->ts_ticks; + ts2->ts_ltick = ts->ts_ltick; + ts2->ts_ftick = ts->ts_ftick; } void sched_class(struct thread *td, int class) { - struct kseq *kseq; - struct kse *ke; + struct tdq *tdq; + struct td_sched *ts; int nclass; int oclass; @@ -1586,36 +1543,36 @@ sched_class(struct thread *td, int class) nclass = PRI_BASE(class); oclass = PRI_BASE(td->td_pri_class); - ke = td->td_kse; - if ((ke->ke_state != KES_ONRUNQ && - ke->ke_state != KES_THREAD) || ke->ke_runq == NULL) - continue; - kseq = KSEQ_CPU(ke->ke_cpu); + ts = td->td_sched; + if (!((ts->ts_state != TSS_ONRUNQ && + ts->ts_state != TSS_THREAD) || ts->ts_runq == NULL)) { + tdq = TDQ_CPU(ts->ts_cpu); #ifdef SMP - /* - * On SMP if we're on the RUNQ we must adjust the transferable - * count because could be changing to or from an interrupt - * class. - */ - if (ke->ke_state == KES_ONRUNQ) { - if (KSE_CAN_MIGRATE(ke)) { - kseq->ksq_transferable--; - kseq->ksq_group->ksg_transferable--; + /* + * On SMP if we're on the RUNQ we must adjust the transferable + * count because could be changing to or from an interrupt + * class. + */ + if (ts->ts_state == TSS_ONRUNQ) { + if (THREAD_CAN_MIGRATE(ts)) { + tdq->ksq_transferable--; + tdq->ksq_group->ksg_transferable--; + } + if (THREAD_CAN_MIGRATE(ts)) { + tdq->ksq_transferable++; + tdq->ksq_group->ksg_transferable++; + } } - if (KSE_CAN_MIGRATE(ke)) { - kseq->ksq_transferable++; - kseq->ksq_group->ksg_transferable++; - } - } #endif - if (oclass == PRI_TIMESHARE) { - kseq->ksq_load_timeshare--; - kseq_nice_rem(kseq, td->td_proc->p_nice); - } - if (nclass == PRI_TIMESHARE) { - kseq->ksq_load_timeshare++; - kseq_nice_add(kseq, td->td_proc->p_nice); + if (oclass == PRI_TIMESHARE) { + tdq->ksq_load_timeshare--; + tdq_nice_rem(tdq, td->td_proc->p_nice); + } + if (nclass == PRI_TIMESHARE) { + tdq->ksq_load_timeshare++; + tdq_nice_add(tdq, td->td_proc->p_nice); + } } td->td_pri_class = class; @@ -1637,17 +1594,44 @@ sched_exit(struct proc *p, struct thread *childtd) parent->td_sched->skg_runtime += childtd->td_sched->skg_runtime; sched_interact_update(parent); - kseq_load_rem(KSEQ_CPU(childtd->td_kse->ke_cpu), childtd->td_kse); + tdq_load_rem(TDQ_CPU(childtd->td_sched->ts_cpu), childtd->td_sched); +} + +void +sched_exit_thread(struct thread *td, struct thread *childtd) +{ +} + +void +sched_userret(struct thread *td) +{ + /* + * XXX we cheat slightly on the locking here to avoid locking in + * the usual case. Setting td_priority here is essentially an + * incomplete workaround for not setting it properly elsewhere. + * Now that some interrupt handlers are threads, not setting it + * properly elsewhere can clobber it in the window between setting + * it here and returning to user mode, so don't waste time setting + * it perfectly here. + */ + KASSERT((td->td_flags & TDF_BORROWING) == 0, + ("thread with borrowed priority returning to userland")); + if (td->td_priority != td->td_user_pri) { + mtx_lock_spin(&sched_lock); + td->td_priority = td->td_user_pri; + td->td_base_pri = td->td_user_pri; + mtx_unlock_spin(&sched_lock); + } } void sched_clock(struct thread *td) { - struct kseq *kseq; - struct kse *ke; + struct tdq *tdq; + struct td_sched *ts; mtx_assert(&sched_lock, MA_OWNED); - kseq = KSEQ_SELF(); + tdq = TDQ_SELF(); #ifdef SMP if (ticks >= bal_tick) sched_balance(); @@ -1657,18 +1641,18 @@ sched_clock(struct thread *td) * We could have been assigned a non real-time thread without an * IPI. */ - if (kseq->ksq_assigned) - kseq_assign(kseq); /* Potentially sets NEEDRESCHED */ + if (tdq->ksq_assigned) + tdq_assign(tdq); /* Potentially sets NEEDRESCHED */ #endif - ke = td->td_kse; + ts = td->td_sched; /* Adjust ticks for pctcpu */ - ke->ke_ticks++; - ke->ke_ltick = ticks; + ts->ts_ticks++; + ts->ts_ltick = ticks; /* Go up to one second beyond our max and then trim back down */ - if (ke->ke_ftick + SCHED_CPU_TICKS + hz < ke->ke_ltick) - sched_pctcpu_update(ke); + if (ts->ts_ftick + SCHED_CPU_TICKS + hz < ts->ts_ltick) + sched_pctcpu_update(ts); if (td->td_flags & TDF_IDLETD) return; @@ -1687,76 +1671,76 @@ sched_clock(struct thread *td) /* * We used up one time slice. */ - if (--ke->ke_slice > 0) + if (--ts->ts_slice > 0) return; /* * We're out of time, recompute priorities and requeue. */ - kseq_load_rem(kseq, ke); + tdq_load_rem(tdq, ts); sched_priority(td); - sched_slice(ke); - if (SCHED_CURR(td, ke)) - ke->ke_runq = kseq->ksq_curr; + sched_slice(ts); + if (SCHED_CURR(td, ts)) + ts->ts_runq = tdq->ksq_curr; else - ke->ke_runq = kseq->ksq_next; - kseq_load_add(kseq, ke); + ts->ts_runq = tdq->ksq_next; + tdq_load_add(tdq, ts); td->td_flags |= TDF_NEEDRESCHED; } int sched_runnable(void) { - struct kseq *kseq; + struct tdq *tdq; int load; load = 1; - kseq = KSEQ_SELF(); + tdq = TDQ_SELF(); #ifdef SMP - if (kseq->ksq_assigned) { + if (tdq->ksq_assigned) { mtx_lock_spin(&sched_lock); - kseq_assign(kseq); + tdq_assign(tdq); mtx_unlock_spin(&sched_lock); } #endif if ((curthread->td_flags & TDF_IDLETD) != 0) { - if (kseq->ksq_load > 0) + if (tdq->ksq_load > 0) goto out; } else - if (kseq->ksq_load - 1 > 0) + if (tdq->ksq_load - 1 > 0) goto out; load = 0; out: return (load); } -struct kse * +struct td_sched * sched_choose(void) { - struct kseq *kseq; - struct kse *ke; + struct tdq *tdq; + struct td_sched *ts; mtx_assert(&sched_lock, MA_OWNED); - kseq = KSEQ_SELF(); + tdq = TDQ_SELF(); #ifdef SMP restart: - if (kseq->ksq_assigned) - kseq_assign(kseq); + if (tdq->ksq_assigned) + tdq_assign(tdq); #endif - ke = kseq_choose(kseq); - if (ke) { + ts = tdq_choose(tdq); + if (ts) { #ifdef SMP - if (ke->ke_thread->td_pri_class == PRI_IDLE) - if (kseq_idled(kseq) == 0) + if (ts->ts_thread->td_pri_class == PRI_IDLE) + if (tdq_idled(tdq) == 0) goto restart; #endif - kseq_runq_rem(kseq, ke); - ke->ke_state = KES_THREAD; - ke->ke_flags &= ~KEF_PREEMPTED; - return (ke); + tdq_runq_rem(tdq, ts); + ts->ts_state = TSS_THREAD; + ts->ts_flags &= ~TSF_PREEMPTED; + return (ts); } #ifdef SMP - if (kseq_idled(kseq) == 0) + if (tdq_idled(tdq) == 0) goto restart; #endif return (NULL); @@ -1765,8 +1749,8 @@ sched_choose(void) void sched_add(struct thread *td, int flags) { - struct kseq *kseq; - struct kse *ke; + struct tdq *tdq; + struct td_sched *ts; int preemptive; int canmigrate; int class; @@ -1775,60 +1759,60 @@ sched_add(struct thread *td, int flags) td, td->td_proc->p_comm, td->td_priority, curthread, curthread->td_proc->p_comm); mtx_assert(&sched_lock, MA_OWNED); - ke = td->td_kse; + ts = td->td_sched; canmigrate = 1; preemptive = !(flags & SRQ_YIELDING); class = PRI_BASE(td->td_pri_class); - kseq = KSEQ_SELF(); - ke->ke_flags &= ~KEF_INTERNAL; + tdq = TDQ_SELF(); + ts->ts_flags &= ~TSF_INTERNAL; #ifdef SMP - if (ke->ke_flags & KEF_ASSIGNED) { - if (ke->ke_flags & KEF_REMOVED) - ke->ke_flags &= ~KEF_REMOVED; + if (ts->ts_flags & TSF_ASSIGNED) { + if (ts->ts_flags & TSF_REMOVED) + ts->ts_flags &= ~TSF_REMOVED; return; } - canmigrate = KSE_CAN_MIGRATE(ke); + canmigrate = THREAD_CAN_MIGRATE(ts); /* * Don't migrate running threads here. Force the long term balancer * to do it. */ - if (ke->ke_flags & KEF_HOLD) { - ke->ke_flags &= ~KEF_HOLD; + if (ts->ts_flags & TSF_HOLD) { + ts->ts_flags &= ~TSF_HOLD; canmigrate = 0; } #endif - KASSERT(ke->ke_state != KES_ONRUNQ, - ("sched_add: kse %p (%s) already in run queue", ke, + KASSERT(ts->ts_state != TSS_ONRUNQ, + ("sched_add: thread %p (%s) already in run queue", td, td->td_proc->p_comm)); KASSERT(td->td_proc->p_sflag & PS_INMEM, ("sched_add: process swapped out")); - KASSERT(ke->ke_runq == NULL, - ("sched_add: KSE %p is still assigned to a run queue", ke)); + KASSERT(ts->ts_runq == NULL, + ("sched_add: thread %p is still assigned to a run queue", td)); if (flags & SRQ_PREEMPTED) - ke->ke_flags |= KEF_PREEMPTED; + ts->ts_flags |= TSF_PREEMPTED; switch (class) { case PRI_ITHD: case PRI_REALTIME: - ke->ke_runq = kseq->ksq_curr; - ke->ke_slice = SCHED_SLICE_MAX; + ts->ts_runq = tdq->ksq_curr; + ts->ts_slice = SCHED_SLICE_MAX; if (canmigrate) - ke->ke_cpu = PCPU_GET(cpuid); + ts->ts_cpu = PCPU_GET(cpuid); break; case PRI_TIMESHARE: - if (SCHED_CURR(td, ke)) - ke->ke_runq = kseq->ksq_curr; + if (SCHED_CURR(td, ts)) + ts->ts_runq = tdq->ksq_curr; else - ke->ke_runq = kseq->ksq_next; + ts->ts_runq = tdq->ksq_next; break; case PRI_IDLE: /* * This is for priority prop. */ - if (ke->ke_thread->td_priority < PRI_MIN_IDLE) - ke->ke_runq = kseq->ksq_curr; + if (ts->ts_thread->td_priority < PRI_MIN_IDLE) + ts->ts_runq = tdq->ksq_curr; else - ke->ke_runq = &kseq->ksq_idle; - ke->ke_slice = SCHED_SLICE_MIN; + ts->ts_runq = &tdq->ksq_idle; + ts->ts_slice = SCHED_SLICE_MIN; break; default: panic("Unknown pri class."); @@ -1838,9 +1822,9 @@ sched_add(struct thread *td, int flags) /* * If this thread is pinned or bound, notify the target cpu. */ - if (!canmigrate && ke->ke_cpu != PCPU_GET(cpuid) ) { - ke->ke_runq = NULL; - kseq_notify(ke, ke->ke_cpu); + if (!canmigrate && ts->ts_cpu != PCPU_GET(cpuid) ) { + ts->ts_runq = NULL; + tdq_notify(ts, ts->ts_cpu); return; } /* @@ -1848,72 +1832,72 @@ sched_add(struct thread *td, int flags) * the global bitmap. If not, see if we should transfer this thread. */ if ((class == PRI_TIMESHARE || class == PRI_REALTIME) && - (kseq->ksq_group->ksg_idlemask & PCPU_GET(cpumask)) != 0) { + (tdq->ksq_group->ksg_idlemask & PCPU_GET(cpumask)) != 0) { /* * Check to see if our group is unidling, and if so, remove it * from the global idle mask. */ - if (kseq->ksq_group->ksg_idlemask == - kseq->ksq_group->ksg_cpumask) - atomic_clear_int(&kseq_idle, kseq->ksq_group->ksg_mask); + if (tdq->ksq_group->ksg_idlemask == + tdq->ksq_group->ksg_cpumask) + atomic_clear_int(&tdq_idle, tdq->ksq_group->ksg_mask); /* * Now remove ourselves from the group specific idle mask. */ - kseq->ksq_group->ksg_idlemask &= ~PCPU_GET(cpumask); - } else if (canmigrate && kseq->ksq_load > 1 && class != PRI_ITHD) - if (kseq_transfer(kseq, ke, class)) + tdq->ksq_group->ksg_idlemask &= ~PCPU_GET(cpumask); + } else if (canmigrate && tdq->ksq_load > 1 && class != PRI_ITHD) + if (tdq_transfer(tdq, ts, class)) return; - ke->ke_cpu = PCPU_GET(cpuid); + ts->ts_cpu = PCPU_GET(cpuid); #endif if (td->td_priority < curthread->td_priority && - ke->ke_runq == kseq->ksq_curr) + ts->ts_runq == tdq->ksq_curr) curthread->td_flags |= TDF_NEEDRESCHED; if (preemptive && maybe_preempt(td)) return; - ke->ke_state = KES_ONRUNQ; + ts->ts_state = TSS_ONRUNQ; - kseq_runq_add(kseq, ke, flags); - kseq_load_add(kseq, ke); + tdq_runq_add(tdq, ts, flags); + tdq_load_add(tdq, ts); } void sched_rem(struct thread *td) { - struct kseq *kseq; - struct kse *ke; + struct tdq *tdq; + struct td_sched *ts; CTR5(KTR_SCHED, "sched_rem: %p(%s) prio %d by %p(%s)", td, td->td_proc->p_comm, td->td_priority, curthread, curthread->td_proc->p_comm); mtx_assert(&sched_lock, MA_OWNED); - ke = td->td_kse; - ke->ke_flags &= ~KEF_PREEMPTED; - if (ke->ke_flags & KEF_ASSIGNED) { - ke->ke_flags |= KEF_REMOVED; + ts = td->td_sched; + ts->ts_flags &= ~TSF_PREEMPTED; + if (ts->ts_flags & TSF_ASSIGNED) { + ts->ts_flags |= TSF_REMOVED; return; } - KASSERT((ke->ke_state == KES_ONRUNQ), - ("sched_rem: KSE not on run queue")); + KASSERT((ts->ts_state == TSS_ONRUNQ), + ("sched_rem: thread not on run queue")); - ke->ke_state = KES_THREAD; - kseq = KSEQ_CPU(ke->ke_cpu); - kseq_runq_rem(kseq, ke); - kseq_load_rem(kseq, ke); + ts->ts_state = TSS_THREAD; + tdq = TDQ_CPU(ts->ts_cpu); + tdq_runq_rem(tdq, ts); + tdq_load_rem(tdq, ts); } fixpt_t sched_pctcpu(struct thread *td) { fixpt_t pctcpu; - struct kse *ke; + struct td_sched *ts; pctcpu = 0; - ke = td->td_kse; - if (ke == NULL) + ts = td->td_sched; + if (ts == NULL) return (0); mtx_lock_spin(&sched_lock); - if (ke->ke_ticks) { + if (ts->ts_ticks) { int rtick; /* @@ -1921,15 +1905,15 @@ sched_pctcpu(struct thread *td) * this causes the cpu usage to decay away too quickly due to * rounding errors. */ - if (ke->ke_ftick + SCHED_CPU_TICKS < ke->ke_ltick || - ke->ke_ltick < (ticks - (hz / 2))) - sched_pctcpu_update(ke); + if (ts->ts_ftick + SCHED_CPU_TICKS < ts->ts_ltick || + ts->ts_ltick < (ticks - (hz / 2))) + sched_pctcpu_update(ts); /* How many rtick per second ? */ - rtick = min(ke->ke_ticks / SCHED_CPU_TIME, SCHED_CPU_TICKS); + rtick = min(ts->ts_ticks / SCHED_CPU_TIME, SCHED_CPU_TICKS); pctcpu = (FSCALE * ((FSCALE * rtick)/realstathz)) >> FSHIFT; } - td->td_proc->p_swtime = ke->ke_ltick - ke->ke_ftick; + td->td_proc->p_swtime = ts->ts_ltick - ts->ts_ftick; mtx_unlock_spin(&sched_lock); return (pctcpu); @@ -1938,18 +1922,18 @@ sched_pctcpu(struct thread *td) void sched_bind(struct thread *td, int cpu) { - struct kse *ke; + struct td_sched *ts; mtx_assert(&sched_lock, MA_OWNED); - ke = td->td_kse; - ke->ke_flags |= KEF_BOUND; + ts = td->td_sched; + ts->ts_flags |= TSF_BOUND; #ifdef SMP if (PCPU_GET(cpuid) == cpu) return; /* sched_rem without the runq_remove */ - ke->ke_state = KES_THREAD; - kseq_load_rem(KSEQ_CPU(ke->ke_cpu), ke); - kseq_notify(ke, cpu); + ts->ts_state = TSS_THREAD; + tdq_load_rem(TDQ_CPU(ts->ts_cpu), ts); + tdq_notify(ts, cpu); /* When we return from mi_switch we'll be on the correct cpu. */ mi_switch(SW_VOL, NULL); #endif @@ -1959,30 +1943,21 @@ void sched_unbind(struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); - td->td_kse->ke_flags &= ~KEF_BOUND; + td->td_sched->ts_flags &= ~TSF_BOUND; } int sched_is_bound(struct thread *td) { mtx_assert(&sched_lock, MA_OWNED); - return (td->td_kse->ke_flags & KEF_BOUND); + return (td->td_sched->ts_flags & TSF_BOUND); } void sched_relinquish(struct thread *td) { -#ifdef KSE - struct ksegrp *kg; - - kg = td->td_ksegrp; -#endif mtx_lock_spin(&sched_lock); -#ifdef KSE - if (kg->kg_pri_class == PRI_TIMESHARE) -#else if (td->td_pri_class == PRI_TIMESHARE) -#endif sched_prio(td, PRI_MAX_TIMESHARE); mi_switch(SW_VOL, NULL); mtx_unlock_spin(&sched_lock); @@ -1997,10 +1972,10 @@ sched_load(void) total = 0; for (i = 0; i <= ksg_maxid; i++) - total += KSEQ_GROUP(i)->ksg_load; + total += TDQ_GROUP(i)->ksg_load; return (total); #else - return (KSEQ_SELF()->ksq_sysload); + return (TDQ_SELF()->ksq_sysload); #endif } diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c index a6eca023cc3a..5e7446ce0f09 100644 --- a/sys/kern/subr_trap.c +++ b/sys/kern/subr_trap.c @@ -149,9 +149,6 @@ ast(struct trapframe *framep) { struct thread *td; struct proc *p; -#ifdef KSE - struct ksegrp *kg; -#endif struct rlimit rlim; int sflag; int flags; @@ -163,9 +160,6 @@ ast(struct trapframe *framep) td = curthread; p = td->td_proc; -#ifdef KSE - kg = td->td_ksegrp; -#endif CTR3(KTR_SYSC, "ast: thread %p (pid %d, %s)", td, p->p_pid, p->p_comm); @@ -204,7 +198,7 @@ ast(struct trapframe *framep) /* * XXXKSE While the fact that we owe a user profiling - * tick is stored per KSE in this code, the statistics + * tick is stored per thread in this code, the statistics * themselves are still stored per process. * This should probably change, by which I mean that * possibly the location of both might change. @@ -264,11 +258,7 @@ ast(struct trapframe *framep) ktrcsw(1, 1); #endif mtx_lock_spin(&sched_lock); -#ifdef KSE - sched_prio(td, kg->kg_user_pri); -#else sched_prio(td, td->td_user_pri); -#endif mi_switch(SW_INVOL, NULL); mtx_unlock_spin(&sched_lock); #ifdef KTRACE diff --git a/sys/kern/tty.c b/sys/kern/tty.c index bd7cf8d423f0..5e7c5969fa10 100644 --- a/sys/kern/tty.c +++ b/sys/kern/tty.c @@ -2578,18 +2578,8 @@ ttyinfo(struct tty *tp) if (proc_compare(pick, p)) pick = p; - td = FIRST_THREAD_IN_PROC(pick); /* XXXKSE */ -#if 0 - KASSERT(td != NULL, ("ttyinfo: no thread")); -#else - if (td == NULL) { - mtx_unlock_spin(&sched_lock); - PGRP_UNLOCK(tp->t_pgrp); - ttyprintf(tp, "foreground process without thread\n"); - tp->t_rocount = 0; - return; - } -#endif + /*^T can only show state for 1 thread. just pick the first. */ + td = FIRST_THREAD_IN_PROC(pick); stateprefix = ""; if (TD_IS_RUNNING(td)) state = "running"; @@ -2669,11 +2659,7 @@ proc_compare(struct proc *p1, struct proc *p2) { int esta, estb; -#ifdef KSE - struct ksegrp *kg; -#else struct thread *td; -#endif mtx_assert(&sched_lock, MA_OWNED); if (p1 == NULL) return (1); @@ -2694,19 +2680,10 @@ proc_compare(struct proc *p1, struct proc *p2) * tie - favor one with highest recent cpu utilization */ esta = estb = 0; -#ifdef KSE - FOREACH_KSEGRP_IN_PROC(p1,kg) { - esta += kg->kg_estcpu; - } - FOREACH_KSEGRP_IN_PROC(p2,kg) { - estb += kg->kg_estcpu; - } -#else FOREACH_THREAD_IN_PROC(p1, td) esta += td->td_estcpu; FOREACH_THREAD_IN_PROC(p2, td) estb += td->td_estcpu; -#endif if (estb > esta) return (1); if (esta > estb) diff --git a/sys/pc98/pc98/machdep.c b/sys/pc98/pc98/machdep.c index c12c8d479e1e..1bd8c251e0ae 100644 --- a/sys/pc98/pc98/machdep.c +++ b/sys/pc98/pc98/machdep.c @@ -1906,11 +1906,7 @@ init386(first) * This may be done better later if it gets more high level * components in it. If so just link td->td_proc here. */ -#ifdef KSE - proc_linkup(&proc0, &ksegrp0, &thread0); -#else proc_linkup(&proc0, &thread0); -#endif /* * Initialize DMAC diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c index 48d5b1b7b39d..5b158d099c19 100644 --- a/sys/powerpc/aim/machdep.c +++ b/sys/powerpc/aim/machdep.c @@ -295,11 +295,7 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp) /* * Start initializing proc0 and thread0. */ -#ifdef KSE - proc_linkup(&proc0, &ksegrp0, &thread0); -#else proc_linkup(&proc0, &thread0); -#endif thread0.td_frame = &frame0; /* diff --git a/sys/powerpc/powerpc/machdep.c b/sys/powerpc/powerpc/machdep.c index 48d5b1b7b39d..5b158d099c19 100644 --- a/sys/powerpc/powerpc/machdep.c +++ b/sys/powerpc/powerpc/machdep.c @@ -295,11 +295,7 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp) /* * Start initializing proc0 and thread0. */ -#ifdef KSE - proc_linkup(&proc0, &ksegrp0, &thread0); -#else proc_linkup(&proc0, &thread0); -#endif thread0.td_frame = &frame0; /* diff --git a/sys/sparc64/sparc64/machdep.c b/sys/sparc64/sparc64/machdep.c index 84eea0b50734..5cf29d528b1f 100644 --- a/sys/sparc64/sparc64/machdep.c +++ b/sys/sparc64/sparc64/machdep.c @@ -391,11 +391,7 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec) /* * Initialize proc0 stuff (p_contested needs to be done early). */ -#ifdef KSE - proc_linkup(&proc0, &ksegrp0, &thread0); -#else proc_linkup(&proc0, &thread0); -#endif proc0.p_md.md_sigtramp = NULL; proc0.p_md.md_utrap = NULL; thread0.td_kstack = kstack0; diff --git a/sys/sun4v/sun4v/machdep.c b/sys/sun4v/sun4v/machdep.c index b76eb864655f..6f3d0dbfcff1 100644 --- a/sys/sun4v/sun4v/machdep.c +++ b/sys/sun4v/sun4v/machdep.c @@ -343,11 +343,7 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec) * Initialize proc0 stuff (p_contested needs to be done early). */ -#ifdef KSE - proc_linkup(&proc0, &ksegrp0, &thread0); -#else proc_linkup(&proc0, &thread0); -#endif proc0.p_md.md_sigtramp = NULL; proc0.p_md.md_utrap = NULL; frame0.tf_tstate = TSTATE_IE | TSTATE_PEF | TSTATE_PRIV; diff --git a/sys/sys/proc.h b/sys/sys/proc.h index 99dbfc6fd9e4..47e21bd72195 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -152,41 +152,32 @@ struct pargs { */ struct auditinfo; struct kaudit_record; -#ifdef KSE -struct kg_sched; -#else struct td_sched; -#endif struct nlminfo; struct kaioinfo; struct p_sched; struct proc; struct sleepqueue; -#ifdef KSE -struct td_sched; -#else struct thread; -#endif struct trapframe; struct turnstile; struct mqueue_notifier; -#ifdef KSE /* - * Here we define the three structures used for process information. + * Here we define the two structures used for process information. * * The first is the thread. It might be thought of as a "Kernel * Schedulable Entity Context". * This structure contains all the information as to where a thread of * execution is now, or was when it was suspended, why it was suspended, * and anything else that will be needed to restart it when it is - * rescheduled. Always associated with a KSE when running, but can be - * reassigned to an equivalent KSE when being restarted for - * load balancing. Each of these is associated with a kernel stack - * and a pcb. + * rescheduled. It includesa sscheduler specific substructure that is differnt + * for each scheduler. * - * It is important to remember that a particular thread structure may only - * exist as long as the system call or kernel entrance (e.g. by pagefault) + * M:N notes. + * It is important to remember that when using M:N threading, + * a particular thread structure may only exist as long as + * the system call or kernel entrance (e.g. by pagefault) * which it is currently executing. It should therefore NEVER be referenced * by pointers in long lived structures that live longer than a single * request. If several threads complete their work at the same time, @@ -198,87 +189,37 @@ struct mqueue_notifier; * get one when it needs a new one. There is also a system * cache of free threads. Threads have priority and partake in priority * inheritance schemes. + * + * The second is the proc (process) which owns all the resources of a process + * other than CPU cycles. which are pqarelled out to the threads. */ -struct thread; - -/* - * The KSEGRP is allocated resources across a number of CPUs. - * (Including a number of CPUxQUANTA. It parcels these QUANTA up among - * its threads, each of which should be running in a different CPU. - * BASE priority and total available quanta are properties of a KSEGRP. - * Multiple KSEGRPs in a single process compete against each other - * for total quanta in the same way that a forked child competes against - * its parent process. - */ -struct ksegrp; - -/* - * A process is the owner of all system resources allocated to a task - * except CPU quanta. - * All KSEGs under one process see, and have the same access to, these - * resources (e.g. files, memory, sockets, credential, kqueues). - * A process may compete for CPU cycles on the same basis as a - * forked process cluster by spawning several KSEGRPs. - */ -struct proc; /*************** - * In pictures: + * Threads are the unit of execution With a single run queue used by all processors: - RUNQ: --->KSE---KSE--... SLEEPQ:[]---THREAD---THREAD---THREAD - \ \ []---THREAD - KSEG---THREAD--THREAD--THREAD [] - []---THREAD---THREAD - - (processors run THREADs from the KSEG until they are exhausted or - the KSEG exhausts its quantum) - -With PER-CPU run queues: -KSEs on the separate run queues directly -They would be given priorities calculated from the KSEG. + RUNQ: --->THREAD---THREAD--... SLEEPQ:[]---THREAD---THREAD---THREAD + []---THREAD + [] + []---THREAD---THREAD +With PER-CPU run queues: +it gets more complicated. * *****************/ -#endif -#ifdef KSE /* * Kernel runnable context (thread). * This is what is put to sleep and reactivated. - * The first KSE available in the correct group will run this thread. - * If several are available, use the one on the same CPU as last time. - * When waiting to be run, threads are hung off the KSEGRP in priority order. - * With N runnable and queued KSEs in the KSEGRP, the first N threads - * are linked to them. Other threads are not yet assigned. - */ -#else -/* * Thread context. Processes may have multiple threads. */ -#endif struct thread { struct proc *td_proc; /* (*) Associated process. */ -#ifdef KSE - struct ksegrp *td_ksegrp; /* (*) Associated KSEG. */ -#else - void *was_td_ksegrp; /* Temporary padding. */ -#endif TAILQ_ENTRY(thread) td_plist; /* (*) All threads in this proc. */ -#ifdef KSE - TAILQ_ENTRY(thread) td_kglist; /* (*) All threads in this ksegrp. */ -#else - TAILQ_ENTRY(thread) was_td_kglist; /* Temporary padding. */ -#endif /* The two queues below should someday be merged. */ TAILQ_ENTRY(thread) td_slpq; /* (j) Sleep queue. */ TAILQ_ENTRY(thread) td_lockq; /* (j) Lock queue. */ -#ifdef KSE - TAILQ_ENTRY(thread) td_runq; /* (j/z) Run queue(s). XXXKSE */ -#else - TAILQ_ENTRY(thread) td_runq; /* (j/z) Run queue(s). */ -#endif TAILQ_HEAD(, selinfo) td_selq; /* (p) List of selinfos. */ struct sleepqueue *td_sleepqueue; /* (k) Associated sleep queue. */ @@ -307,23 +248,12 @@ struct thread { struct lock_list_entry *td_sleeplocks; /* (k) Held sleep locks. */ int td_intr_nesting_level; /* (k) Interrupt recursion. */ int td_pinned; /* (k) Temporary cpu pin count. */ -#ifdef KSE struct kse_thr_mailbox *td_mailbox; /* (*) Userland mailbox address. */ -#else - void *was_td_mailbox; /* Temporary padding. */ -#endif struct ucred *td_ucred; /* (k) Reference to credentials. */ -#ifdef KSE struct thread *td_standin; /* (k + a) Use this for an upcall. */ struct kse_upcall *td_upcall; /* (k + j) Upcall structure. */ - u_int new_td_estcpu; /* Temporary padding. */ - u_int new_td_slptime; /* Temporary padding. */ -#else - void *was_td_standin; /* Temporary padding. */ - void *was_td_upcall; /* Temporary padding. */ u_int td_estcpu; /* (j) Sum of the same field in KSEs. */ u_int td_slptime; /* (j) How long completely blocked. */ -#endif u_int td_pticks; /* (k) Statclock hits for profiling */ u_int td_sticks; /* (k) Statclock hits in system mode. */ u_int td_iticks; /* (k) Statclock hits in intr mode. */ @@ -335,11 +265,7 @@ struct thread { sigset_t td_sigmask; /* (c) Current signal mask. */ volatile u_int td_generation; /* (k) For detection of preemption */ stack_t td_sigstk; /* (k) Stack ptr and on-stack flag. */ -#ifdef KSE int td_kflags; /* (c) Flags for KSE threading. */ -#else - int was_td_kflags; /* Temporary padding. */ -#endif int td_xsig; /* (c) Signal for ptrace */ u_long td_profil_addr; /* (k) Temporary addr until AST. */ u_int td_profil_ticks; /* (k) Temporary ticks until AST. */ @@ -350,15 +276,9 @@ struct thread { #define td_startcopy td_endzero u_char td_base_pri; /* (j) Thread base kernel priority. */ u_char td_priority; /* (j) Thread active priority. */ -#ifdef KSE - u_char new_td_pri_class; /* Temporary padding. */ - u_char new_td_user_pri; /* Temporary padding. */ - u_char new_td_base_user_pri; /* Temporary padding. */ -#else u_char td_pri_class; /* (j) Scheduling class. */ u_char td_user_pri; /* (j) User pri from estcpu and nice. */ - u_char td_base_user_pri; /* (j) Base user pri */ -#endif + u_char td_base_user_pri; /* (j) Base user pri */ #define td_endcopy td_pcb /* @@ -427,27 +347,15 @@ struct thread { #define TDP_OLDMASK 0x00000001 /* Need to restore mask after suspend. */ #define TDP_INKTR 0x00000002 /* Thread is currently in KTR code. */ #define TDP_INKTRACE 0x00000004 /* Thread is currently in KTRACE code. */ -#ifdef KSE #define TDP_UPCALLING 0x00000008 /* This thread is doing an upcall. */ -#else -/* 0x00000008 */ -#endif #define TDP_COWINPROGRESS 0x00000010 /* Snapshot copy-on-write in progress. */ #define TDP_ALTSTACK 0x00000020 /* Have alternate signal stack. */ #define TDP_DEADLKTREAT 0x00000040 /* Lock aquisition - deadlock treatment. */ -#ifdef KSE #define TDP_SA 0x00000080 /* A scheduler activation based thread. */ -#else -/* 0x00000080 */ -#endif #define TDP_NOSLEEPING 0x00000100 /* Thread is not allowed to sleep on a sq. */ #define TDP_OWEUPC 0x00000200 /* Call addupc() at next AST. */ #define TDP_ITHREAD 0x00000400 /* Thread is an interrupt thread. */ -#ifdef KSE #define TDP_CAN_UNBIND 0x00000800 /* Only temporarily bound. */ -#else -/* 0x00000800 */ -#endif #define TDP_SCHED1 0x00001000 /* Reserved for scheduler private use */ #define TDP_SCHED2 0x00002000 /* Reserved for scheduler private use */ #define TDP_SCHED3 0x00004000 /* Reserved for scheduler private use */ @@ -467,18 +375,16 @@ struct thread { #define TDI_LOCK 0x0008 /* Stopped on a lock. */ #define TDI_IWAIT 0x0010 /* Awaiting interrupt. */ -#ifdef KSE /* * flags (in kflags) related to M:N threading. */ -#define TDK_KSEREL 0x0001 /* Blocked in msleep on kg->kg_completed. */ +#define TDK_KSEREL 0x0001 /* Blocked in msleep on p->p_completed. */ #define TDK_KSERELSIG 0x0002 /* Blocked in msleep on p->p_siglist. */ #define TDK_WAKEUP 0x0004 /* Thread has been woken by kse_wakeup. */ #define TD_CAN_UNBIND(td) \ (((td)->td_pflags & TDP_CAN_UNBIND) && \ ((td)->td_upcall != NULL)) -#endif #define TD_IS_SLEEPING(td) ((td)->td_inhibitors & TDI_SLEEPING) #define TD_ON_SLEEPQ(td) ((td)->td_wchan != NULL) @@ -520,14 +426,13 @@ struct thread { #define TD_SET_RUNQ(td) (td)->td_state = TDS_RUNQ #define TD_SET_CAN_RUN(td) (td)->td_state = TDS_CAN_RUN -#ifdef KSE /* * An upcall is used when returning to userland. If a thread does not have * an upcall on return to userland the thread exports its context and exits. */ struct kse_upcall { - TAILQ_ENTRY(kse_upcall) ku_link; /* List of upcalls in KSEG. */ - struct ksegrp *ku_ksegrp; /* Associated KSEG. */ + TAILQ_ENTRY(kse_upcall) ku_link; /* List of upcalls in proc. */ + struct proc *ku_proc; /* Associated proc. */ struct thread *ku_owner; /* Owning thread. */ int ku_flags; /* KUF_* flags. */ struct kse_mailbox *ku_mailbox; /* Userland mailbox address. */ @@ -539,38 +444,6 @@ struct kse_upcall { #define KUF_DOUPCALL 0x00001 /* Do upcall now; don't wait. */ #define KUF_EXITING 0x00002 /* Upcall structure is exiting. */ -/* - * Kernel-scheduled entity group (KSEG). The scheduler considers each KSEG to - * be an indivisible unit from a time-sharing perspective, though each KSEG may - * contain multiple KSEs. - */ -struct ksegrp { - struct proc *kg_proc; /* (*) Proc that contains this KSEG. */ - TAILQ_ENTRY(ksegrp) kg_ksegrp; /* (*) Queue of KSEGs in kg_proc. */ - TAILQ_HEAD(, thread) kg_threads;/* (td_kglist) All threads. */ - TAILQ_HEAD(, thread) kg_runq; /* (td_runq) waiting RUNNABLE threads */ - TAILQ_HEAD(, kse_upcall) kg_upcalls; /* All upcalls in the group. */ - -#define kg_startzero kg_estcpu - u_int kg_estcpu; /* (j) Sum of the same field in KSEs. */ - u_int kg_slptime; /* (j) How long completely blocked. */ - int kg_numupcalls; /* (j) Num upcalls. */ - int kg_upsleeps; /* (c) Num threads in kse_release(). */ - struct kse_thr_mailbox *kg_completed; /* (c) Completed thread mboxes. */ - int kg_nextupcall; /* (n) Next upcall time. */ - int kg_upquantum; /* (n) Quantum to schedule an upcall. */ -#define kg_endzero kg_pri_class - -#define kg_startcopy kg_endzero - u_char kg_pri_class; /* (j) Scheduling class. */ - u_char kg_user_pri; /* (j) User pri from estcpu and nice. */ - u_char kg_base_user_pri; /* (j) Base user pri */ -#define kg_endcopy kg_numthreads - int kg_numthreads; /* (j) Num threads in total. */ - struct kg_sched *kg_sched; /* (*) Scheduler-specific data. */ -}; -#endif - /* * XXX: Does this belong in resource.h or resourcevar.h instead? * Resource usage extension. The times in rusage structs in the kernel are @@ -592,18 +465,12 @@ struct rusage_ext { }; /* - * The old fashionned process. May have multiple threads, KSEGRPs - * and KSEs. Starts off with a single embedded KSEGRP and THREAD. + * The old fashionned process. May have multiple threads. + * Starts off with a single embedded THREAD. */ struct proc { LIST_ENTRY(proc) p_list; /* (d) List of all processes. */ -#ifdef KSE - TAILQ_HEAD(, ksegrp) p_ksegrps; /* (c)(kg_ksegrp) All KSEGs. */ -#else - TAILQ_HEAD(, thread) was_p_ksegrps; /* Temporary padding. */ -#endif TAILQ_HEAD(, thread) p_threads; /* (j)(td_plist) Threads. (shortcut) */ - TAILQ_HEAD(, thread) p_suspended; /* (td_runq) Suspended threads. */ struct ucred *p_ucred; /* (c) Process owner's identity. */ struct filedesc *p_fd; /* (b) Open files. */ struct filedesc_to_leader *p_fdtol; /* (b) Tracking node */ @@ -611,6 +478,7 @@ struct proc { struct pstats *p_stats; /* (b) Accounting/statistics (CPU). */ struct plimit *p_limit; /* (c) Process limits. */ struct sigacts *p_sigacts; /* (x) Signal actions, state (CPU). */ + TAILQ_HEAD(, kse_upcall) p_upcalls; /* All upcalls in the proc. */ /* * The following don't make too much sense. @@ -666,6 +534,14 @@ struct proc { int p_boundary_count;/* (c) Num threads at user boundary */ int p_pendingcnt; /* how many signals are pending */ struct itimers *p_itimers; /* (c) POSIX interval timers. */ +/* from ksegrp */ + u_int p_estcpu; /* (j) Sum of the field in threads. */ + u_int p_slptime; /* (j) How long completely blocked. */ + int p_numupcalls; /* (j) Num upcalls. */ + int p_upsleeps; /* (c) Num threads in kse_release(). */ + struct kse_thr_mailbox *p_completed; /* (c) Completed thread mboxes. */ + int p_nextupcall; /* (n) Next upcall time. */ + int p_upquantum; /* (n) Quantum to schedule an upcall. */ /* End area that is zeroed on creation. */ #define p_endzero p_magic @@ -684,11 +560,6 @@ struct proc { u_short p_xstat; /* (c) Exit status; also stop sig. */ struct knlist p_klist; /* (c) Knotes attached to this proc. */ int p_numthreads; /* (j) Number of threads. */ -#ifdef KSE - int p_numksegrps; /* (c) Number of ksegrps. */ -#else - int was_p_numksegrps; /* Temporary padding. */ -#endif struct mdproc p_md; /* Any machine-dependent fields. */ struct callout p_itcallout; /* (h + c) Interval timer callout. */ u_short p_acflag; /* (c) Accounting flags. */ @@ -797,22 +668,13 @@ MALLOC_DECLARE(M_ZOMBIE); #define FOREACH_PROC_IN_SYSTEM(p) \ LIST_FOREACH((p), &allproc, p_list) -#ifdef KSE -#define FOREACH_KSEGRP_IN_PROC(p, kg) \ - TAILQ_FOREACH((kg), &(p)->p_ksegrps, kg_ksegrp) -#define FOREACH_THREAD_IN_GROUP(kg, td) \ - TAILQ_FOREACH((td), &(kg)->kg_threads, td_kglist) -#define FOREACH_UPCALL_IN_GROUP(kg, ku) \ - TAILQ_FOREACH((ku), &(kg)->kg_upcalls, ku_link) -#endif #define FOREACH_THREAD_IN_PROC(p, td) \ TAILQ_FOREACH((td), &(p)->p_threads, td_plist) +#define FOREACH_UPCALL_IN_PROC(p, ku) \ + TAILQ_FOREACH((ku), &(p)->p_upcalls, ku_link) /* XXXKSE the following lines should probably only be used in 1:1 code: */ #define FIRST_THREAD_IN_PROC(p) TAILQ_FIRST(&(p)->p_threads) -#ifdef KSE -#define FIRST_KSEGRP_IN_PROC(p) TAILQ_FIRST(&(p)->p_ksegrps) -#endif /* * We use process IDs <= PID_MAX; PID_MAX + 1 must also fit in a pid_t, @@ -923,9 +785,6 @@ extern u_long pgrphash; extern struct sx allproc_lock; extern struct sx proctree_lock; extern struct mtx ppeers_lock; -#ifdef KSE -extern struct ksegrp ksegrp0; /* Primary ksegrp in proc0. */ -#endif extern struct proc proc0; /* Process slot for swapper. */ extern struct thread thread0; /* Primary thread in proc0. */ extern struct vmspace vmspace0; /* VM space for proc0. */ @@ -976,11 +835,7 @@ void pargs_drop(struct pargs *pa); void pargs_free(struct pargs *pa); void pargs_hold(struct pargs *pa); void procinit(void); -#ifdef KSE -void proc_linkup(struct proc *p, struct ksegrp *kg, struct thread *td); -#else void proc_linkup(struct proc *p, struct thread *td); -#endif void proc_reparent(struct proc *child, struct proc *newparent); struct pstats *pstats_alloc(void); void pstats_fork(struct pstats *src, struct pstats *dst); @@ -1008,11 +863,6 @@ void cpu_fork(struct thread *, struct proc *, struct thread *, int); void cpu_set_fork_handler(struct thread *, void (*)(void *), void *); /* New in KSE. */ -#ifdef KSE -struct ksegrp *ksegrp_alloc(void); -void ksegrp_free(struct ksegrp *kg); -void ksegrp_stash(struct ksegrp *kg); -#endif void kse_GC(void); void kseinit(void); void cpu_set_upcall(struct thread *td, struct thread *td0); @@ -1023,24 +873,14 @@ void cpu_thread_exit(struct thread *); void cpu_thread_setup(struct thread *td); void cpu_thread_swapin(struct thread *); void cpu_thread_swapout(struct thread *); -#ifdef KSE -void ksegrp_link(struct ksegrp *kg, struct proc *p); -void ksegrp_unlink(struct ksegrp *kg); -#endif struct thread *thread_alloc(void); void thread_continued(struct proc *p); void thread_exit(void) __dead2; int thread_export_context(struct thread *td, int willexit); void thread_free(struct thread *td); -#ifdef KSE -void thread_link(struct thread *td, struct ksegrp *kg); -#else void thread_link(struct thread *td, struct proc *p); -#endif void thread_reap(void); -#ifdef KSE struct thread *thread_schedule_upcall(struct thread *td, struct kse_upcall *ku); -#endif void thread_signal_add(struct thread *td, ksiginfo_t *); int thread_single(int how); void thread_single_end(void); @@ -1058,21 +898,17 @@ void thread_unlink(struct thread *td); void thread_unsuspend(struct proc *p); void thread_unsuspend_one(struct thread *td); void thread_unthread(struct thread *td); -#ifdef KSE int thread_userret(struct thread *td, struct trapframe *frame); void thread_user_enter(struct thread *td); -#endif void thread_wait(struct proc *p); struct thread *thread_find(struct proc *p, lwpid_t tid); void thr_exit1(void); -#ifdef KSE struct kse_upcall *upcall_alloc(void); void upcall_free(struct kse_upcall *ku); -void upcall_link(struct kse_upcall *ku, struct ksegrp *kg); +void upcall_link(struct kse_upcall *ku, struct proc *p); void upcall_unlink(struct kse_upcall *ku); void upcall_remove(struct thread *td); void upcall_stash(struct kse_upcall *ke); -#endif #endif /* _KERNEL */ diff --git a/sys/sys/rtprio.h b/sys/sys/rtprio.h index ba02871e8495..9178046507a9 100644 --- a/sys/sys/rtprio.h +++ b/sys/sys/rtprio.h @@ -75,17 +75,11 @@ struct rtprio { }; #ifdef _KERNEL -#ifdef KSE -struct ksegrp; -int rtp_to_pri(struct rtprio *, struct ksegrp *); -void pri_to_rtp(struct ksegrp *, struct rtprio *); -#else struct thread; int rtp_to_pri(struct rtprio *, struct thread *); void pri_to_rtp(struct thread *, struct rtprio *); #endif #endif -#endif #ifndef _KERNEL #include diff --git a/sys/sys/runq.h b/sys/sys/runq.h index 433d0e05e19c..0f3524c12a76 100644 --- a/sys/sys/runq.h +++ b/sys/sys/runq.h @@ -31,7 +31,7 @@ #include -struct kse; +struct td_sched; /* * Run queue parameters. @@ -43,7 +43,7 @@ struct kse; /* * Head of run queues. */ -TAILQ_HEAD(rqhead, kse); +TAILQ_HEAD(rqhead, td_sched); /* * Bit array which maintains the status of a run queue. When a queue is @@ -62,10 +62,10 @@ struct runq { struct rqhead rq_queues[RQ_NQS]; }; -void runq_add(struct runq *, struct kse *, int flags); +void runq_add(struct runq *, struct td_sched *, int flags); int runq_check(struct runq *); -struct kse *runq_choose(struct runq *); +struct td_sched *runq_choose(struct runq *); void runq_init(struct runq *); -void runq_remove(struct runq *, struct kse *); +void runq_remove(struct runq *, struct td_sched *); #endif diff --git a/sys/sys/sched.h b/sys/sys/sched.h index 69e4a0cbe674..a9f174880de6 100644 --- a/sys/sys/sched.h +++ b/sys/sys/sched.h @@ -86,23 +86,15 @@ void sched_fork(struct thread *td, struct thread *childtd); * KSE Groups contain scheduling priority information. They record the * behavior of groups of KSEs and threads. */ -#ifdef KSE -void sched_class(struct ksegrp *kg, int class); -void sched_exit_ksegrp(struct ksegrp *kg, struct thread *childtd); -void sched_fork_ksegrp(struct thread *td, struct ksegrp *child); -#else void sched_class(struct thread *td, int class); -#endif void sched_nice(struct proc *p, int nice); /* * Threads are switched in and out, block on resources, have temporary - * priorities inherited from their ksegs, and use up cpu time. + * priorities inherited from their procs, and use up cpu time. */ -#ifdef KSE void sched_exit_thread(struct thread *td, struct thread *child); void sched_fork_thread(struct thread *td, struct thread *child); -#endif void sched_lend_prio(struct thread *td, u_char prio); void sched_lend_user_prio(struct thread *td, u_char pri); fixpt_t sched_pctcpu(struct thread *td); @@ -111,11 +103,7 @@ void sched_sleep(struct thread *td); void sched_switch(struct thread *td, struct thread *newtd, int flags); void sched_unlend_prio(struct thread *td, u_char prio); void sched_unlend_user_prio(struct thread *td, u_char pri); -#ifdef KSE -void sched_user_prio(struct ksegrp *kg, u_char prio); -#else void sched_user_prio(struct thread *td, u_char prio); -#endif void sched_userret(struct thread *td); void sched_wakeup(struct thread *td); @@ -142,9 +130,6 @@ int sched_is_bound(struct thread *td); * These procedures tell the process data structure allocation code how * many bytes to actually allocate. */ -#ifdef KSE -int sched_sizeof_ksegrp(void); -#endif int sched_sizeof_proc(void); int sched_sizeof_thread(void); @@ -162,15 +147,11 @@ sched_unpin(void) /* temporarily here */ void schedinit(void); -#ifdef KSE -void sched_init_concurrency(struct ksegrp *kg); -void sched_set_concurrency(struct ksegrp *kg, int cuncurrency); -#endif +void sched_init_concurrency(struct proc *p); +void sched_set_concurrency(struct proc *p, int cuncurrency); void sched_schedinit(void); -#ifdef KSE -void sched_newproc(struct proc *p, struct ksegrp *kg, struct thread *td); +void sched_newproc(struct proc *p, struct thread *td); void sched_thread_exit(struct thread *td); -#endif void sched_newthread(struct thread *td); #endif /* _KERNEL */ diff --git a/sys/vm/vm_glue.c b/sys/vm/vm_glue.c index de56199c0b84..ff1d1ff086b0 100644 --- a/sys/vm/vm_glue.c +++ b/sys/vm/vm_glue.c @@ -682,9 +682,6 @@ scheduler(dummy) ppri = INT_MIN; sx_slock(&allproc_lock); FOREACH_PROC_IN_SYSTEM(p) { -#ifdef KSE - struct ksegrp *kg; -#endif if (p->p_sflag & (PS_INMEM | PS_SWAPPINGOUT | PS_SWAPPINGIN)) { continue; } @@ -696,18 +693,13 @@ scheduler(dummy) * */ if (td->td_inhibitors == TDI_SWAPPED) { -#ifdef KSE - kg = td->td_ksegrp; - pri = p->p_swtime + kg->kg_slptime; -#else pri = p->p_swtime + td->td_slptime; -#endif if ((p->p_sflag & PS_SWAPINREQ) == 0) { pri -= p->p_nice * 8; } /* - * if this ksegrp/thread is higher priority + * if this thread is higher priority * and there is enough space, then select * this process instead of the previous * selection. @@ -816,9 +808,6 @@ int action; { struct proc *p; struct thread *td; -#ifdef KSE - struct ksegrp *kg; -#endif int didswap = 0; retry: @@ -892,24 +881,15 @@ int action; * do not swapout a realtime process * Check all the thread groups.. */ -#ifdef KSE - FOREACH_KSEGRP_IN_PROC(p, kg) { - if (PRI_IS_REALTIME(kg->kg_pri_class)) -#else FOREACH_THREAD_IN_PROC(p, td) { if (PRI_IS_REALTIME(td->td_pri_class)) -#endif goto nextproc; /* * Guarantee swap_idle_threshold1 * time in memory. */ -#ifdef KSE - if (kg->kg_slptime < swap_idle_threshold1) -#else if (td->td_slptime < swap_idle_threshold1) -#endif goto nextproc; /* @@ -921,16 +901,9 @@ int action; * This could be refined to support * swapping out a thread. */ -#ifdef KSE - FOREACH_THREAD_IN_GROUP(kg, td) { - if ((td->td_priority) < PSOCK || - !thread_safetoswapout(td)) - goto nextproc; - } -#else - if ((td->td_priority) < PSOCK || !thread_safetoswapout(td)) + if ((td->td_priority) < PSOCK || + !thread_safetoswapout(td)) goto nextproc; -#endif /* * If the system is under memory stress, * or if we are swapping @@ -939,20 +912,11 @@ int action; */ if (((action & VM_SWAP_NORMAL) == 0) && (((action & VM_SWAP_IDLE) == 0) || -#ifdef KSE - (kg->kg_slptime < swap_idle_threshold2))) -#else (td->td_slptime < swap_idle_threshold2))) -#endif goto nextproc; -#ifdef KSE - if (minslptime > kg->kg_slptime) - minslptime = kg->kg_slptime; -#else if (minslptime > td->td_slptime) minslptime = td->td_slptime; -#endif } /* diff --git a/sys/vm/vm_zeroidle.c b/sys/vm/vm_zeroidle.c index 14f47dc925d3..b445003b8810 100644 --- a/sys/vm/vm_zeroidle.c +++ b/sys/vm/vm_zeroidle.c @@ -179,11 +179,7 @@ pagezero_start(void __unused *arg) PROC_UNLOCK(pagezero_proc); mtx_lock_spin(&sched_lock); td = FIRST_THREAD_IN_PROC(pagezero_proc); -#ifdef KSE - sched_class(td->td_ksegrp, PRI_IDLE); -#else sched_class(td, PRI_IDLE); -#endif sched_prio(td, PRI_MAX_IDLE); setrunqueue(td, SRQ_BORING); mtx_unlock_spin(&sched_lock);