Commit 5/14 of sched_lock decomposition.
- Protect the cp_time tick counts with atomics instead of a global lock. There will only be one atomic per tick and this allows all processors to execute softclock concurrently. - In softclock, protect access to rusage and td_*tick data with the thread_lock(), expanding the scope of the thread lock over the whole function. - Do some creative re-arranging in hardclock() to avoid excess locking. - Protect the p_timer fields with the per-process spinlock. Tested by: kris, current@ Tested on: i386, amd64, ULE, 4BSD, libthr, libkse, PREEMPTION, etc. Discussed with: kris, attilio, kmacy, jhb, julian, bde (small parts each)
This commit is contained in:
parent
a54e85fdbf
commit
40acdeabab
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=170297
@ -201,32 +201,35 @@ hardclock_cpu(int usermode)
|
|||||||
struct pstats *pstats;
|
struct pstats *pstats;
|
||||||
struct thread *td = curthread;
|
struct thread *td = curthread;
|
||||||
struct proc *p = td->td_proc;
|
struct proc *p = td->td_proc;
|
||||||
|
int ast;
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Run current process's virtual and profile time, as needed.
|
* Run current process's virtual and profile time, as needed.
|
||||||
*/
|
*/
|
||||||
mtx_lock_spin_flags(&sched_lock, MTX_QUIET);
|
|
||||||
sched_tick();
|
|
||||||
#ifdef KSE
|
|
||||||
#if 0 /* for now do nothing */
|
|
||||||
if (p->p_flag & P_SA) {
|
|
||||||
/* XXXKSE What to do? Should do more. */
|
|
||||||
}
|
|
||||||
#endif
|
|
||||||
#endif
|
|
||||||
pstats = p->p_stats;
|
pstats = p->p_stats;
|
||||||
|
ast = 0;
|
||||||
if (usermode &&
|
if (usermode &&
|
||||||
timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value) &&
|
timevalisset(&pstats->p_timer[ITIMER_VIRTUAL].it_value)) {
|
||||||
itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
|
PROC_SLOCK(p);
|
||||||
p->p_sflag |= PS_ALRMPEND;
|
if (itimerdecr(&pstats->p_timer[ITIMER_VIRTUAL], tick) == 0) {
|
||||||
td->td_flags |= TDF_ASTPENDING;
|
p->p_sflag |= PS_ALRMPEND;
|
||||||
|
ast = 1;
|
||||||
|
}
|
||||||
|
PROC_SUNLOCK(p);
|
||||||
}
|
}
|
||||||
if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value) &&
|
if (timevalisset(&pstats->p_timer[ITIMER_PROF].it_value)) {
|
||||||
itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
|
PROC_SLOCK(p);
|
||||||
p->p_sflag |= PS_PROFPEND;
|
if (itimerdecr(&pstats->p_timer[ITIMER_PROF], tick) == 0) {
|
||||||
td->td_flags |= TDF_ASTPENDING;
|
p->p_sflag |= PS_PROFPEND;
|
||||||
|
ast = 1;
|
||||||
|
}
|
||||||
|
PROC_SUNLOCK(p);
|
||||||
}
|
}
|
||||||
mtx_unlock_spin_flags(&sched_lock, MTX_QUIET);
|
thread_lock(td);
|
||||||
|
sched_tick();
|
||||||
|
if (ast)
|
||||||
|
td->td_flags |= TDF_ASTPENDING;
|
||||||
|
thread_unlock(td);
|
||||||
|
|
||||||
#ifdef HWPMC_HOOKS
|
#ifdef HWPMC_HOOKS
|
||||||
if (PMC_CPU_HAS_SAMPLES(PCPU_GET(cpuid)))
|
if (PMC_CPU_HAS_SAMPLES(PCPU_GET(cpuid)))
|
||||||
@ -272,8 +275,8 @@ hardclock(int usermode, uintfptr_t pc)
|
|||||||
mtx_unlock_spin_flags(&callout_lock, MTX_QUIET);
|
mtx_unlock_spin_flags(&callout_lock, MTX_QUIET);
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* swi_sched acquires sched_lock, so we don't want to call it with
|
* swi_sched acquires the thread lock, so we don't want to call it
|
||||||
* callout_lock held; incorrect locking order.
|
* with callout_lock held; incorrect locking order.
|
||||||
*/
|
*/
|
||||||
if (need_softclock)
|
if (need_softclock)
|
||||||
swi_sched(softclock_ih, 0);
|
swi_sched(softclock_ih, 0);
|
||||||
@ -411,6 +414,7 @@ statclock(int usermode)
|
|||||||
td = curthread;
|
td = curthread;
|
||||||
p = td->td_proc;
|
p = td->td_proc;
|
||||||
|
|
||||||
|
thread_lock_flags(td, MTX_QUIET);
|
||||||
if (usermode) {
|
if (usermode) {
|
||||||
/*
|
/*
|
||||||
* Charge the time as appropriate.
|
* Charge the time as appropriate.
|
||||||
@ -420,11 +424,10 @@ statclock(int usermode)
|
|||||||
thread_statclock(1);
|
thread_statclock(1);
|
||||||
#endif
|
#endif
|
||||||
td->td_uticks++;
|
td->td_uticks++;
|
||||||
mtx_lock_spin_flags(&time_lock, MTX_QUIET);
|
|
||||||
if (p->p_nice > NZERO)
|
if (p->p_nice > NZERO)
|
||||||
cp_time[CP_NICE]++;
|
atomic_add_long(&cp_time[CP_NICE], 1);
|
||||||
else
|
else
|
||||||
cp_time[CP_USER]++;
|
atomic_add_long(&cp_time[CP_USER], 1);
|
||||||
} else {
|
} else {
|
||||||
/*
|
/*
|
||||||
* Came from kernel mode, so we were:
|
* Came from kernel mode, so we were:
|
||||||
@ -441,8 +444,7 @@ statclock(int usermode)
|
|||||||
if ((td->td_pflags & TDP_ITHREAD) ||
|
if ((td->td_pflags & TDP_ITHREAD) ||
|
||||||
td->td_intr_nesting_level >= 2) {
|
td->td_intr_nesting_level >= 2) {
|
||||||
td->td_iticks++;
|
td->td_iticks++;
|
||||||
mtx_lock_spin_flags(&time_lock, MTX_QUIET);
|
atomic_add_long(&cp_time[CP_INTR], 1);
|
||||||
cp_time[CP_INTR]++;
|
|
||||||
} else {
|
} else {
|
||||||
#ifdef KSE
|
#ifdef KSE
|
||||||
if (p->p_flag & P_SA)
|
if (p->p_flag & P_SA)
|
||||||
@ -450,19 +452,12 @@ statclock(int usermode)
|
|||||||
#endif
|
#endif
|
||||||
td->td_pticks++;
|
td->td_pticks++;
|
||||||
td->td_sticks++;
|
td->td_sticks++;
|
||||||
mtx_lock_spin_flags(&time_lock, MTX_QUIET);
|
|
||||||
if (!TD_IS_IDLETHREAD(td))
|
if (!TD_IS_IDLETHREAD(td))
|
||||||
cp_time[CP_SYS]++;
|
atomic_add_long(&cp_time[CP_SYS], 1);
|
||||||
else
|
else
|
||||||
cp_time[CP_IDLE]++;
|
atomic_add_long(&cp_time[CP_IDLE], 1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
mtx_unlock_spin_flags(&time_lock, MTX_QUIET);
|
|
||||||
CTR4(KTR_SCHED, "statclock: %p(%s) prio %d stathz %d",
|
|
||||||
td, td->td_proc->p_comm, td->td_priority, (stathz)?stathz:hz);
|
|
||||||
|
|
||||||
mtx_lock_spin_flags(&sched_lock, MTX_QUIET);
|
|
||||||
sched_clock(td);
|
|
||||||
|
|
||||||
/* Update resource usage integrals and maximums. */
|
/* Update resource usage integrals and maximums. */
|
||||||
MPASS(p->p_vmspace != NULL);
|
MPASS(p->p_vmspace != NULL);
|
||||||
@ -474,7 +469,10 @@ statclock(int usermode)
|
|||||||
rss = pgtok(vmspace_resident_count(vm));
|
rss = pgtok(vmspace_resident_count(vm));
|
||||||
if (ru->ru_maxrss < rss)
|
if (ru->ru_maxrss < rss)
|
||||||
ru->ru_maxrss = rss;
|
ru->ru_maxrss = rss;
|
||||||
mtx_unlock_spin_flags(&sched_lock, MTX_QUIET);
|
CTR4(KTR_SCHED, "statclock: %p(%s) prio %d stathz %d",
|
||||||
|
td, td->td_proc->p_comm, td->td_priority, (stathz)?stathz:hz);
|
||||||
|
sched_clock(td);
|
||||||
|
thread_unlock(td);
|
||||||
}
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
|
Loading…
Reference in New Issue
Block a user