diff --git a/sys/alpha/include/mutex.h b/sys/alpha/include/mutex.h index 9b04128e964a..a6e295d612d0 100644 --- a/sys/alpha/include/mutex.h +++ b/sys/alpha/include/mutex.h @@ -36,8 +36,6 @@ #ifdef _KERNEL -#define mtx_legal2block() \ - ((alpha_pal_rdps() & ALPHA_PSL_IPL_MASK) == ALPHA_PSL_IPL_0) #define mtx_intr_enable(mutex) (mutex)->mtx_saveintr = ALPHA_PSL_IPL_0 /* diff --git a/sys/amd64/include/mutex.h b/sys/amd64/include/mutex.h index a5f88dea236c..4184cde390f9 100644 --- a/sys/amd64/include/mutex.h +++ b/sys/amd64/include/mutex.h @@ -40,7 +40,6 @@ /* Global locks */ extern struct mtx clock_lock; -#define mtx_legal2block() (read_eflags() & PSL_I) #define mtx_intr_enable(mutex) (mutex)->mtx_saveintr |= PSL_I /* diff --git a/sys/i386/include/mutex.h b/sys/i386/include/mutex.h index a5f88dea236c..4184cde390f9 100644 --- a/sys/i386/include/mutex.h +++ b/sys/i386/include/mutex.h @@ -40,7 +40,6 @@ /* Global locks */ extern struct mtx clock_lock; -#define mtx_legal2block() (read_eflags() & PSL_I) #define mtx_intr_enable(mutex) (mutex)->mtx_saveintr |= PSL_I /* diff --git a/sys/ia64/include/mutex.h b/sys/ia64/include/mutex.h index 809a56eb7377..f325f2b307f8 100644 --- a/sys/ia64/include/mutex.h +++ b/sys/ia64/include/mutex.h @@ -38,7 +38,6 @@ #ifdef _KERNEL -#define mtx_legal2block() (ia64_get_psr() & IA64_PSR_I) #define mtx_intr_enable(mutex) (mutex)->mtx_saveintr |= IA64_PSR_I #endif /* _KERNEL */ diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index c6776de0a23b..2f9b3aa79802 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -406,6 +406,10 @@ fork1(p1, flags, procp) if (p1->p_sflag & PS_PROFIL) startprofclock(p2); mtx_unlock_spin(&sched_lock); + /* + * We start off holding one spinlock after fork: sched_lock. + */ + p2->p_spinlocks = 1; PROC_UNLOCK(p2); MALLOC(p2->p_cred, struct pcred *, sizeof(struct pcred), M_SUBPROC, M_WAITOK); diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index 1ca5cca6eaab..6540b561b66e 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -1094,6 +1094,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) } PCPU_SET(witness_spin_check, i | w->w_level); mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); + p->p_spinlocks++; + MPASS(p->p_spinlocks > 0); w->w_file = file; w->w_line = line; m->mtx_line = line; @@ -1116,7 +1118,7 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) if (cold) goto out; - if (!mtx_legal2block()) + if (p->p_spinlocks != 0) panic("blockable mtx_lock() of %s when not legal @ %s:%d", m->mtx_description, file, line); /* @@ -1263,10 +1265,12 @@ void witness_exit(struct mtx *m, int flags, const char *file, int line) { struct witness *w; + struct proc *p; if (witness_cold || m->mtx_witness == NULL || panicstr) return; w = m->mtx_witness; + p = curproc; if (flags & MTX_SPIN) { if ((m->mtx_flags & MTX_SPIN) == 0) @@ -1283,6 +1287,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) PCPU_SET(witness_spin_check, PCPU_GET(witness_spin_check) & ~w->w_level); mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); + MPASS(p->p_spinlocks > 0); + p->p_spinlocks--; return; } if ((m->mtx_flags & MTX_SPIN) != 0) @@ -1297,7 +1303,7 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) return; } - if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold) + if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold) panic("switchable mtx_unlock() of %s when not legal @ %s:%d", m->mtx_description, file, line); LIST_REMOVE(m, mtx_held); diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c index 1ca5cca6eaab..6540b561b66e 100644 --- a/sys/kern/subr_turnstile.c +++ b/sys/kern/subr_turnstile.c @@ -1094,6 +1094,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) } PCPU_SET(witness_spin_check, i | w->w_level); mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); + p->p_spinlocks++; + MPASS(p->p_spinlocks > 0); w->w_file = file; w->w_line = line; m->mtx_line = line; @@ -1116,7 +1118,7 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) if (cold) goto out; - if (!mtx_legal2block()) + if (p->p_spinlocks != 0) panic("blockable mtx_lock() of %s when not legal @ %s:%d", m->mtx_description, file, line); /* @@ -1263,10 +1265,12 @@ void witness_exit(struct mtx *m, int flags, const char *file, int line) { struct witness *w; + struct proc *p; if (witness_cold || m->mtx_witness == NULL || panicstr) return; w = m->mtx_witness; + p = curproc; if (flags & MTX_SPIN) { if ((m->mtx_flags & MTX_SPIN) == 0) @@ -1283,6 +1287,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) PCPU_SET(witness_spin_check, PCPU_GET(witness_spin_check) & ~w->w_level); mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); + MPASS(p->p_spinlocks > 0); + p->p_spinlocks--; return; } if ((m->mtx_flags & MTX_SPIN) != 0) @@ -1297,7 +1303,7 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) return; } - if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold) + if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold) panic("switchable mtx_unlock() of %s when not legal @ %s:%d", m->mtx_description, file, line); LIST_REMOVE(m, mtx_held); diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index 1ca5cca6eaab..6540b561b66e 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -1094,6 +1094,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) } PCPU_SET(witness_spin_check, i | w->w_level); mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); + p->p_spinlocks++; + MPASS(p->p_spinlocks > 0); w->w_file = file; w->w_line = line; m->mtx_line = line; @@ -1116,7 +1118,7 @@ witness_enter(struct mtx *m, int flags, const char *file, int line) if (cold) goto out; - if (!mtx_legal2block()) + if (p->p_spinlocks != 0) panic("blockable mtx_lock() of %s when not legal @ %s:%d", m->mtx_description, file, line); /* @@ -1263,10 +1265,12 @@ void witness_exit(struct mtx *m, int flags, const char *file, int line) { struct witness *w; + struct proc *p; if (witness_cold || m->mtx_witness == NULL || panicstr) return; w = m->mtx_witness; + p = curproc; if (flags & MTX_SPIN) { if ((m->mtx_flags & MTX_SPIN) == 0) @@ -1283,6 +1287,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) PCPU_SET(witness_spin_check, PCPU_GET(witness_spin_check) & ~w->w_level); mtx_unlock_spin_flags(&w_mtx, MTX_QUIET); + MPASS(p->p_spinlocks > 0); + p->p_spinlocks--; return; } if ((m->mtx_flags & MTX_SPIN) != 0) @@ -1297,7 +1303,7 @@ witness_exit(struct mtx *m, int flags, const char *file, int line) return; } - if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold) + if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold) panic("switchable mtx_unlock() of %s when not legal @ %s:%d", m->mtx_description, file, line); LIST_REMOVE(m, mtx_held); diff --git a/sys/powerpc/include/mutex.h b/sys/powerpc/include/mutex.h index 9b04128e964a..a6e295d612d0 100644 --- a/sys/powerpc/include/mutex.h +++ b/sys/powerpc/include/mutex.h @@ -36,8 +36,6 @@ #ifdef _KERNEL -#define mtx_legal2block() \ - ((alpha_pal_rdps() & ALPHA_PSL_IPL_MASK) == ALPHA_PSL_IPL_0) #define mtx_intr_enable(mutex) (mutex)->mtx_saveintr = ALPHA_PSL_IPL_0 /* diff --git a/sys/sys/proc.h b/sys/sys/proc.h index e96dffa0b7f3..b77d49ec2077 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -213,6 +213,7 @@ struct proc { struct vnode *p_textvp; /* (b) Vnode of executable. */ struct mtx p_mtx; /* (k) Lock for this struct. */ + u_int p_spinlocks; /* (k) Count of held spin locks. */ char p_lock; /* (c) Process lock (prevent swap) count. */ u_char p_oncpu; /* (j) Which cpu we are on. */ u_char p_lastcpu; /* (j) Last cpu we were on. */