From 3db720fdce718d51446544af6ee7512f54fed29b Mon Sep 17 00:00:00 2001 From: David Xu Date: Fri, 25 Aug 2006 06:12:53 +0000 Subject: [PATCH] Add user priority loaning code to support priority propagation for 1:1 threading's POSIX priority mutexes, the code is no-op unless priority-aware umtx code is committed. --- sys/kern/kern_umtx.c | 5 ++++ sys/kern/sched_4bsd.c | 57 +++++++++++++++++++++++++++++++++++- sys/kern/sched_core.c | 67 +++++++++++++++++++++++++++++++++++++++---- sys/kern/sched_ule.c | 58 ++++++++++++++++++++++++++++++++++++- sys/sys/sched.h | 5 +++- sys/sys/umtx.h | 2 +- 6 files changed, 184 insertions(+), 10 deletions(-) diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c index cb248a855f5c..c6db80c4055c 100644 --- a/sys/kern/kern_umtx.c +++ b/sys/kern/kern_umtx.c @@ -601,6 +601,11 @@ kern_umtx_wake(struct thread *td, void *uaddr, int n_wake) return (0); } +void +umtx_pi_adjust(struct thread *td __unused, u_char oldpri __unused) +{ +} + int _umtx_lock(struct thread *td, struct _umtx_lock_args *uap) /* struct umtx *umtx */ diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c index 1e34e804244f..ce7582fbae27 100644 --- a/sys/kern/sched_4bsd.c +++ b/sys/kern/sched_4bsd.c @@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -586,7 +587,7 @@ resetpriority(struct ksegrp *kg) NICE_WEIGHT * (kg->kg_proc->p_nice - PRIO_MIN); newpriority = min(max(newpriority, PRI_MIN_TIMESHARE), PRI_MAX_TIMESHARE); - kg->kg_user_pri = newpriority; + sched_user_prio(kg, newpriority); } } @@ -863,6 +864,60 @@ sched_prio(struct thread *td, u_char prio) turnstile_adjust(td, oldprio); } +void +sched_user_prio(struct ksegrp *kg, u_char prio) +{ + struct thread *td; + u_char oldprio; + + kg->kg_base_user_pri = prio; + + /* XXXKSE only for 1:1 */ + + td = TAILQ_FIRST(&kg->kg_threads); + if (td == NULL) { + kg->kg_user_pri = prio; + return; + } + + if (td->td_flags & TDF_UBORROWING && kg->kg_user_pri <= prio) + return; + + oldprio = kg->kg_user_pri; + kg->kg_user_pri = prio; + + if (TD_ON_UPILOCK(td) && oldprio != prio) + umtx_pi_adjust(td, oldprio); +} + +void +sched_lend_user_prio(struct thread *td, u_char prio) +{ + u_char oldprio; + + td->td_flags |= TDF_UBORROWING; + + oldprio = td->td_ksegrp->kg_user_pri; + td->td_ksegrp->kg_user_pri = prio; + + if (TD_ON_UPILOCK(td) && oldprio != prio) + umtx_pi_adjust(td, oldprio); +} + +void +sched_unlend_user_prio(struct thread *td, u_char prio) +{ + struct ksegrp *kg = td->td_ksegrp; + u_char base_pri; + + base_pri = kg->kg_base_user_pri; + if (prio >= base_pri) { + td->td_flags &= ~TDF_UBORROWING; + sched_user_prio(kg, base_pri); + } else + sched_lend_user_prio(td, prio); +} + void sched_sleep(struct thread *td) { diff --git a/sys/kern/sched_core.c b/sys/kern/sched_core.c index f0c4fc27d298..b2de346f0881 100644 --- a/sys/kern/sched_core.c +++ b/sys/kern/sched_core.c @@ -49,6 +49,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #ifdef KTRACE @@ -633,7 +634,7 @@ sched_calc_pri(struct ksegrp *kg) pri = PUSER_MAX; return (pri); } - return (kg->kg_user_pri); + return (kg->kg_base_user_pri); } static int @@ -646,7 +647,7 @@ sched_recalc_pri(struct kse *ke, uint64_t now) kg = ke->ke_ksegrp; delta = now - ke->ke_timestamp; if (__predict_false(!sched_is_timeshare(kg))) - return (kg->kg_user_pri); + return (kg->kg_base_user_pri); if (delta > NS_MAX_SLEEP_TIME) sleep_time = NS_MAX_SLEEP_TIME; @@ -947,6 +948,60 @@ sched_prio(struct thread *td, u_char prio) turnstile_adjust(td, oldprio); } +void +sched_user_prio(struct ksegrp *kg, u_char prio) +{ + struct thread *td; + u_char oldprio; + + kg->kg_base_user_pri = prio; + + /* XXXKSE only for 1:1 */ + + td = TAILQ_FIRST(&kg->kg_threads); + if (td == NULL) { + kg->kg_user_pri = prio; + return; + } + + if (td->td_flags & TDF_UBORROWING && kg->kg_user_pri <= prio) + return; + + oldprio = kg->kg_user_pri; + kg->kg_user_pri = prio; + + if (TD_ON_UPILOCK(td) && oldprio != prio) + umtx_pi_adjust(td, oldprio); +} + +void +sched_lend_user_prio(struct thread *td, u_char prio) +{ + u_char oldprio; + + td->td_flags |= TDF_UBORROWING; + + oldprio = td->td_ksegrp->kg_user_pri; + td->td_ksegrp->kg_user_pri = prio; + + if (TD_ON_UPILOCK(td) && oldprio != prio) + umtx_pi_adjust(td, oldprio); +} + +void +sched_unlend_user_prio(struct thread *td, u_char prio) +{ + struct ksegrp *kg = td->td_ksegrp; + u_char base_pri; + + base_pri = kg->kg_base_user_pri; + if (prio >= base_pri) { + td->td_flags &= ~TDF_UBORROWING; + sched_user_prio(kg, base_pri); + } else + sched_lend_user_prio(td, prio); +} + void sched_switch(struct thread *td, struct thread *newtd, int flags) { @@ -1038,7 +1093,7 @@ sched_nice(struct proc *p, int nice) p->p_nice = nice; FOREACH_KSEGRP_IN_PROC(p, kg) { if (kg->kg_pri_class == PRI_TIMESHARE) { - kg->kg_user_pri = sched_calc_pri(kg); + sched_user_prio(kg, sched_calc_pri(kg)); FOREACH_THREAD_IN_GROUP(kg, td) td->td_flags |= TDF_NEEDRESCHED; } @@ -1082,7 +1137,7 @@ sched_wakeup(struct thread *td) now = now - mykseq->ksq_last_timestamp + kseq->ksq_last_timestamp; #endif - kg->kg_user_pri = sched_recalc_pri(ke, now); + sched_user_prio(kg, sched_recalc_pri(ke, now)); } } setrunqueue(td, SRQ_BORING); @@ -1109,7 +1164,7 @@ sched_fork_ksegrp(struct thread *td, struct ksegrp *child) mtx_assert(&sched_lock, MA_OWNED); child->kg_slptime = kg->kg_slptime * CHILD_WEIGHT / 100; if (child->kg_pri_class == PRI_TIMESHARE) - child->kg_user_pri = sched_calc_pri(child); + sched_user_prio(child, sched_calc_pri(child)); kg->kg_slptime = kg->kg_slptime * PARENT_WEIGHT / 100; } @@ -1275,7 +1330,7 @@ sched_tick(void) td->td_flags |= TDF_NEEDRESCHED; sched_update_runtime(ke, now); sched_commit_runtime(ke); - kg->kg_user_pri = sched_calc_pri(kg); + sched_user_prio(kg, sched_calc_pri(kg)); ke->ke_slice = sched_timeslice(ke); ke->ke_flags &= ~KEF_FIRST_SLICE; if (ke->ke_flags & KEF_BOUND || td->td_pinned) { diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c index a352445b209f..0c7729904acc 100644 --- a/sys/kern/sched_ule.c +++ b/sys/kern/sched_ule.c @@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #ifdef KTRACE #include @@ -1060,7 +1061,7 @@ sched_priority(struct ksegrp *kg) else if (pri < PRI_MIN_TIMESHARE) pri = PRI_MIN_TIMESHARE; - kg->kg_user_pri = pri; + sched_user_prio(kg, pri); return; } @@ -1343,6 +1344,60 @@ sched_prio(struct thread *td, u_char prio) turnstile_adjust(td, oldprio); } +void +sched_user_prio(struct ksegrp *kg, u_char prio) +{ + struct thread *td; + u_char oldprio; + + kg->kg_base_user_pri = prio; + + /* XXXKSE only for 1:1 */ + + td = TAILQ_FIRST(&kg->kg_threads); + if (td == NULL) { + kg->kg_user_pri = prio; + return; + } + + if (td->td_flags & TDF_UBORROWING && kg->kg_user_pri <= prio) + return; + + oldprio = kg->kg_user_pri; + kg->kg_user_pri = prio; + + if (TD_ON_UPILOCK(td) && oldprio != prio) + umtx_pi_adjust(td, oldprio); +} + +void +sched_lend_user_prio(struct thread *td, u_char prio) +{ + u_char oldprio; + + td->td_flags |= TDF_UBORROWING; + + oldprio = td->td_ksegrp->kg_user_pri; + td->td_ksegrp->kg_user_pri = prio; + + if (TD_ON_UPILOCK(td) && oldprio != prio) + umtx_pi_adjust(td, oldprio); +} + +void +sched_unlend_user_prio(struct thread *td, u_char prio) +{ + struct ksegrp *kg = td->td_ksegrp; + u_char base_pri; + + base_pri = kg->kg_base_user_pri; + if (prio >= base_pri) { + td->td_flags &= ~TDF_UBORROWING; + sched_user_prio(kg, base_pri); + } else + sched_lend_user_prio(td, prio); +} + void sched_switch(struct thread *td, struct thread *newtd, int flags) { @@ -1519,6 +1574,7 @@ sched_fork_ksegrp(struct thread *td, struct ksegrp *child) child->kg_slptime = kg->kg_slptime; child->kg_runtime = kg->kg_runtime; child->kg_user_pri = kg->kg_user_pri; + child->kg_base_user_pri = kg->kg_base_user_pri; sched_interact_fork(child); kg->kg_runtime += tickincr; sched_interact_update(kg); diff --git a/sys/sys/sched.h b/sys/sys/sched.h index b782c1f62079..ccb335d88736 100644 --- a/sys/sys/sched.h +++ b/sys/sys/sched.h @@ -63,12 +63,15 @@ void sched_nice(struct proc *p, int nice); */ void sched_exit_thread(struct thread *td, struct thread *child); void sched_fork_thread(struct thread *td, struct thread *child); +void sched_lend_prio(struct thread *td, u_char prio); +void sched_lend_user_prio(struct thread *td, u_char pri); fixpt_t sched_pctcpu(struct thread *td); void sched_prio(struct thread *td, u_char prio); -void sched_lend_prio(struct thread *td, u_char prio); void sched_sleep(struct thread *td); void sched_switch(struct thread *td, struct thread *newtd, int flags); void sched_unlend_prio(struct thread *td, u_char prio); +void sched_unlend_user_prio(struct thread *td, u_char pri); +void sched_user_prio(struct ksegrp *kg, u_char prio); void sched_userret(struct thread *td); void sched_wakeup(struct thread *td); diff --git a/sys/sys/umtx.h b/sys/sys/umtx.h index 57389fdcf55c..796ee47626a4 100644 --- a/sys/sys/umtx.h +++ b/sys/sys/umtx.h @@ -142,6 +142,6 @@ struct umtx_q *umtxq_alloc(void); void umtxq_free(struct umtx_q *); struct thread; int kern_umtx_wake(struct thread *td, void *uaddr, int n_wake); - +void umtx_pi_adjust(struct thread *td, u_char oldpri); #endif /* !_KERNEL */ #endif /* !_SYS_UMTX_H_ */