2007-11-08 14:47:55 +00:00
|
|
|
/*-
|
|
|
|
* Copyright (c) 2007 Stephan Uphoff <ups@FreeBSD.org>
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
* 3. Neither the name of the author nor the names of any co-contributors
|
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
* without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Machine independent bits of reader/writer lock implementation.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <sys/cdefs.h>
|
|
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
|
|
|
|
#include "opt_ddb.h"
|
2009-05-26 20:28:22 +00:00
|
|
|
#include "opt_kdtrace.h"
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
#include <sys/param.h>
|
|
|
|
#include <sys/systm.h>
|
|
|
|
|
|
|
|
#include <sys/kernel.h>
|
|
|
|
#include <sys/ktr.h>
|
|
|
|
#include <sys/lock.h>
|
|
|
|
#include <sys/mutex.h>
|
|
|
|
#include <sys/proc.h>
|
|
|
|
#include <sys/rmlock.h>
|
|
|
|
#include <sys/sched.h>
|
|
|
|
#include <sys/smp.h>
|
|
|
|
#include <sys/turnstile.h>
|
|
|
|
#include <sys/lock_profile.h>
|
|
|
|
#include <machine/cpu.h>
|
|
|
|
|
|
|
|
#ifdef DDB
|
|
|
|
#include <ddb/ddb.h>
|
|
|
|
#endif
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
#define RMPF_ONQUEUE 1
|
|
|
|
#define RMPF_SIGNAL 2
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/*
|
|
|
|
* To support usage of rmlock in CVs and msleep yet another list for the
|
|
|
|
* priority tracker would be needed. Using this lock for cv and msleep also
|
|
|
|
* does not seem very useful
|
2007-11-08 14:47:55 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
static __inline void compiler_memory_barrier(void) {
|
|
|
|
__asm __volatile("":::"memory");
|
|
|
|
}
|
|
|
|
|
2011-11-16 21:51:17 +00:00
|
|
|
static void assert_rm(const struct lock_object *lock, int what);
|
2009-04-26 21:16:03 +00:00
|
|
|
static void lock_rm(struct lock_object *lock, int how);
|
2009-05-26 20:28:22 +00:00
|
|
|
#ifdef KDTRACE_HOOKS
|
2011-11-16 21:51:17 +00:00
|
|
|
static int owner_rm(const struct lock_object *lock, struct thread **owner);
|
2009-05-26 20:28:22 +00:00
|
|
|
#endif
|
2009-04-26 21:16:03 +00:00
|
|
|
static int unlock_rm(struct lock_object *lock);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
struct lock_class lock_class_rm = {
|
|
|
|
.lc_name = "rm",
|
|
|
|
.lc_flags = LC_SLEEPLOCK | LC_RECURSABLE,
|
2007-11-18 14:43:53 +00:00
|
|
|
.lc_assert = assert_rm,
|
2007-11-08 14:47:55 +00:00
|
|
|
#if 0
|
|
|
|
#ifdef DDB
|
|
|
|
.lc_ddb_show = db_show_rwlock,
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
.lc_lock = lock_rm,
|
|
|
|
.lc_unlock = unlock_rm,
|
2009-05-26 20:28:22 +00:00
|
|
|
#ifdef KDTRACE_HOOKS
|
|
|
|
.lc_owner = owner_rm,
|
|
|
|
#endif
|
2007-11-08 14:47:55 +00:00
|
|
|
};
|
|
|
|
|
2007-11-18 14:43:53 +00:00
|
|
|
static void
|
2011-11-16 21:51:17 +00:00
|
|
|
assert_rm(const struct lock_object *lock, int what)
|
2007-11-18 14:43:53 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
panic("assert_rm called");
|
|
|
|
}
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
static void
|
2009-04-26 21:16:03 +00:00
|
|
|
lock_rm(struct lock_object *lock, int how)
|
|
|
|
{
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
panic("lock_rm called");
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2009-04-26 21:16:03 +00:00
|
|
|
unlock_rm(struct lock_object *lock)
|
|
|
|
{
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
panic("unlock_rm called");
|
|
|
|
}
|
|
|
|
|
2009-05-26 20:28:22 +00:00
|
|
|
#ifdef KDTRACE_HOOKS
|
|
|
|
static int
|
2011-11-16 21:51:17 +00:00
|
|
|
owner_rm(const struct lock_object *lock, struct thread **owner)
|
2009-05-26 20:28:22 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
panic("owner_rm called");
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
static struct mtx rm_spinlock;
|
|
|
|
|
|
|
|
MTX_SYSINIT(rm_spinlock, &rm_spinlock, "rm_spinlock", MTX_SPIN);
|
|
|
|
|
|
|
|
/*
|
2009-12-25 01:16:24 +00:00
|
|
|
* Add or remove tracker from per-cpu list.
|
2009-04-26 21:16:03 +00:00
|
|
|
*
|
2009-12-25 01:16:24 +00:00
|
|
|
* The per-cpu list can be traversed at any time in forward direction from an
|
2009-04-26 21:16:03 +00:00
|
|
|
* interrupt on the *local* cpu.
|
2007-11-08 14:47:55 +00:00
|
|
|
*/
|
2009-04-26 21:16:03 +00:00
|
|
|
static void inline
|
|
|
|
rm_tracker_add(struct pcpu *pc, struct rm_priotracker *tracker)
|
|
|
|
{
|
|
|
|
struct rm_queue *next;
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
/* Initialize all tracker pointers */
|
|
|
|
tracker->rmp_cpuQueue.rmq_prev = &pc->pc_rm_queue;
|
|
|
|
next = pc->pc_rm_queue.rmq_next;
|
|
|
|
tracker->rmp_cpuQueue.rmq_next = next;
|
2009-04-26 21:16:03 +00:00
|
|
|
|
|
|
|
/* rmq_prev is not used during froward traversal. */
|
2007-11-08 14:47:55 +00:00
|
|
|
next->rmq_prev = &tracker->rmp_cpuQueue;
|
2009-04-26 21:16:03 +00:00
|
|
|
|
|
|
|
/* Update pointer to first element. */
|
2009-12-25 21:14:05 +00:00
|
|
|
pc->pc_rm_queue.rmq_next = &tracker->rmp_cpuQueue;
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
static void inline
|
|
|
|
rm_tracker_remove(struct pcpu *pc, struct rm_priotracker *tracker)
|
|
|
|
{
|
2007-11-08 14:47:55 +00:00
|
|
|
struct rm_queue *next, *prev;
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
next = tracker->rmp_cpuQueue.rmq_next;
|
|
|
|
prev = tracker->rmp_cpuQueue.rmq_prev;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/* Not used during forward traversal. */
|
|
|
|
next->rmq_prev = prev;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/* Remove from list. */
|
|
|
|
prev->rmq_next = next;
|
|
|
|
}
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
static void
|
|
|
|
rm_cleanIPI(void *arg)
|
|
|
|
{
|
2007-11-08 14:47:55 +00:00
|
|
|
struct pcpu *pc;
|
2009-04-26 21:16:03 +00:00
|
|
|
struct rmlock *rm = arg;
|
2007-11-08 14:47:55 +00:00
|
|
|
struct rm_priotracker *tracker;
|
2009-04-26 21:16:03 +00:00
|
|
|
struct rm_queue *queue;
|
2007-11-08 14:47:55 +00:00
|
|
|
pc = pcpu_find(curcpu);
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue;
|
2007-11-08 14:47:55 +00:00
|
|
|
queue = queue->rmq_next) {
|
2009-04-26 21:16:03 +00:00
|
|
|
tracker = (struct rm_priotracker *)queue;
|
|
|
|
if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) {
|
2007-11-08 14:47:55 +00:00
|
|
|
tracker->rmp_flags = RMPF_ONQUEUE;
|
|
|
|
mtx_lock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
|
2007-11-08 14:47:55 +00:00
|
|
|
rmp_qentry);
|
|
|
|
mtx_unlock_spin(&rm_spinlock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
CTASSERT((RM_SLEEPABLE & LO_CLASSFLAGS) == RM_SLEEPABLE);
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
void
|
2009-05-29 10:52:37 +00:00
|
|
|
rm_init_flags(struct rmlock *rm, const char *name, int opts)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
2009-05-29 10:52:37 +00:00
|
|
|
int liflags;
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2009-05-29 10:52:37 +00:00
|
|
|
liflags = 0;
|
|
|
|
if (!(opts & RM_NOWITNESS))
|
|
|
|
liflags |= LO_WITNESS;
|
|
|
|
if (opts & RM_RECURSE)
|
|
|
|
liflags |= LO_RECURSABLE;
|
2010-09-01 19:50:03 +00:00
|
|
|
rm->rm_writecpus = all_cpus;
|
2007-11-08 14:47:55 +00:00
|
|
|
LIST_INIT(&rm->rm_activeReaders);
|
2010-09-01 19:50:03 +00:00
|
|
|
if (opts & RM_SLEEPABLE) {
|
|
|
|
liflags |= RM_SLEEPABLE;
|
|
|
|
sx_init_flags(&rm->rm_lock_sx, "rmlock_sx", SX_RECURSE);
|
|
|
|
} else
|
|
|
|
mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx", MTX_NOWITNESS);
|
2009-05-29 10:52:37 +00:00
|
|
|
lock_init(&rm->lock_object, &lock_class_rm, name, NULL, liflags);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
rm_init(struct rmlock *rm, const char *name)
|
|
|
|
{
|
|
|
|
|
|
|
|
rm_init_flags(rm, name, 0);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
rm_destroy(struct rmlock *rm)
|
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
sx_destroy(&rm->rm_lock_sx);
|
|
|
|
else
|
|
|
|
mtx_destroy(&rm->rm_lock_mtx);
|
2007-11-08 14:47:55 +00:00
|
|
|
lock_destroy(&rm->lock_object);
|
|
|
|
}
|
|
|
|
|
2007-11-10 15:06:30 +00:00
|
|
|
int
|
2011-11-16 21:51:17 +00:00
|
|
|
rm_wowned(const struct rmlock *rm)
|
2007-11-10 15:06:30 +00:00
|
|
|
{
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
return (sx_xlocked(&rm->rm_lock_sx));
|
|
|
|
else
|
|
|
|
return (mtx_owned(&rm->rm_lock_mtx));
|
2007-11-10 15:06:30 +00:00
|
|
|
}
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
void
|
|
|
|
rm_sysinit(void *arg)
|
|
|
|
{
|
|
|
|
struct rm_args *args = arg;
|
2009-05-29 10:52:37 +00:00
|
|
|
|
|
|
|
rm_init(args->ra_rm, args->ra_desc);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
rm_sysinit_flags(void *arg)
|
|
|
|
{
|
|
|
|
struct rm_args_flags *args = arg;
|
|
|
|
|
|
|
|
rm_init_flags(args->ra_rm, args->ra_desc, args->ra_opts);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
static int
|
|
|
|
_rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
|
|
|
struct pcpu *pc;
|
|
|
|
struct rm_queue *queue;
|
2009-04-26 21:16:03 +00:00
|
|
|
struct rm_priotracker *atracker;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
critical_enter();
|
|
|
|
pc = pcpu_find(curcpu);
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/* Check if we just need to do a proper critical_exit. */
|
2011-06-13 13:28:31 +00:00
|
|
|
if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) {
|
2007-11-08 14:47:55 +00:00
|
|
|
critical_exit();
|
2010-09-01 19:50:03 +00:00
|
|
|
return (1);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
2009-12-25 01:16:24 +00:00
|
|
|
/* Remove our tracker from the per-cpu list. */
|
2009-04-26 21:16:03 +00:00
|
|
|
rm_tracker_remove(pc, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/* Check to see if the IPI granted us the lock after all. */
|
|
|
|
if (tracker->rmp_flags) {
|
|
|
|
/* Just add back tracker - we hold the lock. */
|
|
|
|
rm_tracker_add(pc, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
critical_exit();
|
2010-09-01 19:50:03 +00:00
|
|
|
return (1);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2009-04-26 21:16:03 +00:00
|
|
|
* We allow readers to aquire a lock even if a writer is blocked if
|
|
|
|
* the lock is recursive and the reader already holds the lock.
|
2007-11-08 14:47:55 +00:00
|
|
|
*/
|
|
|
|
if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) {
|
|
|
|
/*
|
2009-12-25 01:16:24 +00:00
|
|
|
* Just grant the lock if this thread already has a tracker
|
|
|
|
* for this lock on the per-cpu queue.
|
2007-11-08 14:47:55 +00:00
|
|
|
*/
|
2009-04-26 21:16:03 +00:00
|
|
|
for (queue = pc->pc_rm_queue.rmq_next;
|
2009-12-25 21:14:05 +00:00
|
|
|
queue != &pc->pc_rm_queue; queue = queue->rmq_next) {
|
2009-04-26 21:16:03 +00:00
|
|
|
atracker = (struct rm_priotracker *)queue;
|
|
|
|
if ((atracker->rmp_rmlock == rm) &&
|
|
|
|
(atracker->rmp_thread == tracker->rmp_thread)) {
|
2007-11-08 14:47:55 +00:00
|
|
|
mtx_lock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
LIST_INSERT_HEAD(&rm->rm_activeReaders,
|
|
|
|
tracker, rmp_qentry);
|
2007-11-08 14:47:55 +00:00
|
|
|
tracker->rmp_flags = RMPF_ONQUEUE;
|
|
|
|
mtx_unlock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
rm_tracker_add(pc, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
critical_exit();
|
2010-09-01 19:50:03 +00:00
|
|
|
return (1);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
sched_unpin();
|
|
|
|
critical_exit();
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (trylock) {
|
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE) {
|
|
|
|
if (!sx_try_xlock(&rm->rm_lock_sx))
|
|
|
|
return (0);
|
|
|
|
} else {
|
|
|
|
if (!mtx_trylock(&rm->rm_lock_mtx))
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
sx_xlock(&rm->rm_lock_sx);
|
|
|
|
else
|
|
|
|
mtx_lock(&rm->rm_lock_mtx);
|
|
|
|
}
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
critical_enter();
|
2007-11-08 14:47:55 +00:00
|
|
|
pc = pcpu_find(curcpu);
|
2011-06-13 13:28:31 +00:00
|
|
|
CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus);
|
2009-04-26 21:16:03 +00:00
|
|
|
rm_tracker_add(pc, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
sched_pin();
|
|
|
|
critical_exit();
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
sx_xunlock(&rm->rm_lock_sx);
|
|
|
|
else
|
|
|
|
mtx_unlock(&rm->rm_lock_mtx);
|
|
|
|
|
|
|
|
return (1);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
int
|
|
|
|
_rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
|
|
|
struct thread *td = curthread;
|
|
|
|
struct pcpu *pc;
|
|
|
|
|
panic: add a switch and infrastructure for stopping other CPUs in SMP case
Historical behavior of letting other CPUs merily go on is a default for
time being. The new behavior can be switched on via
kern.stop_scheduler_on_panic tunable and sysctl.
Stopping of the CPUs has (at least) the following benefits:
- more of the system state at panic time is preserved intact
- threads and interrupts do not interfere with dumping of the system
state
Only one thread runs uninterrupted after panic if stop_scheduler_on_panic
is set. That thread might call code that is also used in normal context
and that code might use locks to prevent concurrent execution of certain
parts. Those locks might be held by the stopped threads and would never
be released. To work around this issue, it was decided that instead of
explicit checks for panic context, we would rather put those checks
inside the locking primitives.
This change has substantial portions written and re-written by attilio
and kib at various times. Other changes are heavily based on the ideas
and patches submitted by jhb and mdf. bde has provided many insights
into the details and history of the current code.
The new behavior may cause problems for systems that use a USB keyboard
for interfacing with system console. This is because of some unusual
locking patterns in the ukbd code which have to be used because on one
hand ukbd is below syscons, but on the other hand it has to interface
with other usb code that uses regular mutexes/Giant for its concurrency
protection. Dumping to USB-connected disks may also be affected.
PR: amd64/139614 (at least)
In cooperation with: attilio, jhb, kib, mdf
Discussed with: arch@, bde
Tested by: Eugene Grosbein <eugen@grosbein.net>,
gnn,
Steven Hartland <killing@multiplay.co.uk>,
glebius,
Andrew Boyer <aboyer@averesystems.com>
(various versions of the patch)
MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
|
|
|
if (SCHEDULER_STOPPED())
|
|
|
|
return (1);
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
tracker->rmp_flags = 0;
|
|
|
|
tracker->rmp_thread = td;
|
|
|
|
tracker->rmp_rmlock = rm;
|
|
|
|
|
|
|
|
td->td_critnest++; /* critical_enter(); */
|
|
|
|
|
|
|
|
compiler_memory_barrier();
|
|
|
|
|
|
|
|
pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
rm_tracker_add(pc, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-05-29 14:20:10 +00:00
|
|
|
sched_pin();
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
compiler_memory_barrier();
|
|
|
|
|
|
|
|
td->td_critnest--;
|
2009-04-26 21:16:03 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Fast path to combine two common conditions into a single
|
|
|
|
* conditional jump.
|
2007-11-08 14:47:55 +00:00
|
|
|
*/
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
if (0 == (td->td_owepreempt |
|
2011-06-13 13:28:31 +00:00
|
|
|
CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)))
|
2010-09-01 19:50:03 +00:00
|
|
|
return (1);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/* We do not have a read token and need to acquire one. */
|
2010-09-01 19:50:03 +00:00
|
|
|
return _rm_rlock_hard(rm, tracker, trylock);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2009-04-26 21:16:03 +00:00
|
|
|
_rm_unlock_hard(struct thread *td,struct rm_priotracker *tracker)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
if (td->td_owepreempt) {
|
|
|
|
td->td_critnest++;
|
|
|
|
critical_exit();
|
|
|
|
}
|
2009-04-26 21:16:03 +00:00
|
|
|
|
|
|
|
if (!tracker->rmp_flags)
|
2007-11-08 14:47:55 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
mtx_lock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
LIST_REMOVE(tracker, rmp_qentry);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
if (tracker->rmp_flags & RMPF_SIGNAL) {
|
|
|
|
struct rmlock *rm;
|
2009-04-26 21:16:03 +00:00
|
|
|
struct turnstile *ts;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
rm = tracker->rmp_rmlock;
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
turnstile_chain_lock(&rm->lock_object);
|
|
|
|
mtx_unlock_spin(&rm_spinlock);
|
|
|
|
|
|
|
|
ts = turnstile_lookup(&rm->lock_object);
|
|
|
|
|
|
|
|
turnstile_signal(ts, TS_EXCLUSIVE_QUEUE);
|
|
|
|
turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
|
|
|
|
turnstile_chain_unlock(&rm->lock_object);
|
|
|
|
} else
|
|
|
|
mtx_unlock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
}
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
void
|
2009-04-26 21:16:03 +00:00
|
|
|
_rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
|
|
|
struct pcpu *pc;
|
|
|
|
struct thread *td = tracker->rmp_thread;
|
|
|
|
|
panic: add a switch and infrastructure for stopping other CPUs in SMP case
Historical behavior of letting other CPUs merily go on is a default for
time being. The new behavior can be switched on via
kern.stop_scheduler_on_panic tunable and sysctl.
Stopping of the CPUs has (at least) the following benefits:
- more of the system state at panic time is preserved intact
- threads and interrupts do not interfere with dumping of the system
state
Only one thread runs uninterrupted after panic if stop_scheduler_on_panic
is set. That thread might call code that is also used in normal context
and that code might use locks to prevent concurrent execution of certain
parts. Those locks might be held by the stopped threads and would never
be released. To work around this issue, it was decided that instead of
explicit checks for panic context, we would rather put those checks
inside the locking primitives.
This change has substantial portions written and re-written by attilio
and kib at various times. Other changes are heavily based on the ideas
and patches submitted by jhb and mdf. bde has provided many insights
into the details and history of the current code.
The new behavior may cause problems for systems that use a USB keyboard
for interfacing with system console. This is because of some unusual
locking patterns in the ukbd code which have to be used because on one
hand ukbd is below syscons, but on the other hand it has to interface
with other usb code that uses regular mutexes/Giant for its concurrency
protection. Dumping to USB-connected disks may also be affected.
PR: amd64/139614 (at least)
In cooperation with: attilio, jhb, kib, mdf
Discussed with: arch@, bde
Tested by: Eugene Grosbein <eugen@grosbein.net>,
gnn,
Steven Hartland <killing@multiplay.co.uk>,
glebius,
Andrew Boyer <aboyer@averesystems.com>
(various versions of the patch)
MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
|
|
|
if (SCHEDULER_STOPPED())
|
|
|
|
return;
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
td->td_critnest++; /* critical_enter(); */
|
|
|
|
pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */
|
2009-04-26 21:16:03 +00:00
|
|
|
rm_tracker_remove(pc, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
td->td_critnest--;
|
2009-05-29 14:20:10 +00:00
|
|
|
sched_unpin();
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
if (0 == (td->td_owepreempt | tracker->rmp_flags))
|
2007-11-08 14:47:55 +00:00
|
|
|
return;
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
_rm_unlock_hard(td, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
_rm_wlock(struct rmlock *rm)
|
|
|
|
{
|
|
|
|
struct rm_priotracker *prio;
|
|
|
|
struct turnstile *ts;
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
cpuset_t readcpus;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
panic: add a switch and infrastructure for stopping other CPUs in SMP case
Historical behavior of letting other CPUs merily go on is a default for
time being. The new behavior can be switched on via
kern.stop_scheduler_on_panic tunable and sysctl.
Stopping of the CPUs has (at least) the following benefits:
- more of the system state at panic time is preserved intact
- threads and interrupts do not interfere with dumping of the system
state
Only one thread runs uninterrupted after panic if stop_scheduler_on_panic
is set. That thread might call code that is also used in normal context
and that code might use locks to prevent concurrent execution of certain
parts. Those locks might be held by the stopped threads and would never
be released. To work around this issue, it was decided that instead of
explicit checks for panic context, we would rather put those checks
inside the locking primitives.
This change has substantial portions written and re-written by attilio
and kib at various times. Other changes are heavily based on the ideas
and patches submitted by jhb and mdf. bde has provided many insights
into the details and history of the current code.
The new behavior may cause problems for systems that use a USB keyboard
for interfacing with system console. This is because of some unusual
locking patterns in the ukbd code which have to be used because on one
hand ukbd is below syscons, but on the other hand it has to interface
with other usb code that uses regular mutexes/Giant for its concurrency
protection. Dumping to USB-connected disks may also be affected.
PR: amd64/139614 (at least)
In cooperation with: attilio, jhb, kib, mdf
Discussed with: arch@, bde
Tested by: Eugene Grosbein <eugen@grosbein.net>,
gnn,
Steven Hartland <killing@multiplay.co.uk>,
glebius,
Andrew Boyer <aboyer@averesystems.com>
(various versions of the patch)
MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
|
|
|
if (SCHEDULER_STOPPED())
|
|
|
|
return;
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
sx_xlock(&rm->rm_lock_sx);
|
|
|
|
else
|
|
|
|
mtx_lock(&rm->rm_lock_mtx);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) {
|
2007-11-08 14:47:55 +00:00
|
|
|
/* Get all read tokens back */
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
readcpus = all_cpus;
|
|
|
|
CPU_NAND(&readcpus, &rm->rm_writecpus);
|
2010-09-01 19:50:03 +00:00
|
|
|
rm->rm_writecpus = all_cpus;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/*
|
2010-09-01 19:50:03 +00:00
|
|
|
* Assumes rm->rm_writecpus update is visible on other CPUs
|
2009-04-26 21:16:03 +00:00
|
|
|
* before rm_cleanIPI is called.
|
2007-11-08 14:47:55 +00:00
|
|
|
*/
|
|
|
|
#ifdef SMP
|
2010-09-01 19:50:03 +00:00
|
|
|
smp_rendezvous_cpus(readcpus,
|
|
|
|
smp_no_rendevous_barrier,
|
2007-11-08 14:47:55 +00:00
|
|
|
rm_cleanIPI,
|
2009-04-26 21:16:03 +00:00
|
|
|
smp_no_rendevous_barrier,
|
|
|
|
rm);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
#else
|
|
|
|
rm_cleanIPI(rm);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
mtx_lock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) {
|
2007-11-08 14:47:55 +00:00
|
|
|
ts = turnstile_trywait(&rm->lock_object);
|
|
|
|
prio->rmp_flags = RMPF_ONQUEUE | RMPF_SIGNAL;
|
|
|
|
mtx_unlock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
turnstile_wait(ts, prio->rmp_thread,
|
|
|
|
TS_EXCLUSIVE_QUEUE);
|
2007-11-08 14:47:55 +00:00
|
|
|
mtx_lock_spin(&rm_spinlock);
|
|
|
|
}
|
|
|
|
mtx_unlock_spin(&rm_spinlock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
_rm_wunlock(struct rmlock *rm)
|
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
sx_xunlock(&rm->rm_lock_sx);
|
|
|
|
else
|
|
|
|
mtx_unlock(&rm->rm_lock_mtx);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef LOCK_DEBUG
|
|
|
|
|
|
|
|
void _rm_wlock_debug(struct rmlock *rm, const char *file, int line)
|
|
|
|
{
|
|
|
|
|
panic: add a switch and infrastructure for stopping other CPUs in SMP case
Historical behavior of letting other CPUs merily go on is a default for
time being. The new behavior can be switched on via
kern.stop_scheduler_on_panic tunable and sysctl.
Stopping of the CPUs has (at least) the following benefits:
- more of the system state at panic time is preserved intact
- threads and interrupts do not interfere with dumping of the system
state
Only one thread runs uninterrupted after panic if stop_scheduler_on_panic
is set. That thread might call code that is also used in normal context
and that code might use locks to prevent concurrent execution of certain
parts. Those locks might be held by the stopped threads and would never
be released. To work around this issue, it was decided that instead of
explicit checks for panic context, we would rather put those checks
inside the locking primitives.
This change has substantial portions written and re-written by attilio
and kib at various times. Other changes are heavily based on the ideas
and patches submitted by jhb and mdf. bde has provided many insights
into the details and history of the current code.
The new behavior may cause problems for systems that use a USB keyboard
for interfacing with system console. This is because of some unusual
locking patterns in the ukbd code which have to be used because on one
hand ukbd is below syscons, but on the other hand it has to interface
with other usb code that uses regular mutexes/Giant for its concurrency
protection. Dumping to USB-connected disks may also be affected.
PR: amd64/139614 (at least)
In cooperation with: attilio, jhb, kib, mdf
Discussed with: arch@, bde
Tested by: Eugene Grosbein <eugen@grosbein.net>,
gnn,
Steven Hartland <killing@multiplay.co.uk>,
glebius,
Andrew Boyer <aboyer@averesystems.com>
(various versions of the patch)
MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
|
|
|
if (SCHEDULER_STOPPED())
|
|
|
|
return;
|
|
|
|
|
2012-09-12 22:10:53 +00:00
|
|
|
KASSERT(!TD_IS_IDLETHREAD(curthread),
|
|
|
|
("rm_wlock() by idle thread %p on rmlock %s @ %s:%d",
|
|
|
|
curthread, rm->lock_object.lo_name, file, line));
|
2009-04-26 21:16:03 +00:00
|
|
|
WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE,
|
2008-09-10 19:13:30 +00:00
|
|
|
file, line, NULL);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
_rm_wlock(rm);
|
|
|
|
|
|
|
|
LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line);
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
WITNESS_LOCK(&rm->rm_lock_sx.lock_object, LOP_EXCLUSIVE,
|
|
|
|
file, line);
|
|
|
|
else
|
|
|
|
WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
curthread->td_locks++;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
void
|
|
|
|
_rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
panic: add a switch and infrastructure for stopping other CPUs in SMP case
Historical behavior of letting other CPUs merily go on is a default for
time being. The new behavior can be switched on via
kern.stop_scheduler_on_panic tunable and sysctl.
Stopping of the CPUs has (at least) the following benefits:
- more of the system state at panic time is preserved intact
- threads and interrupts do not interfere with dumping of the system
state
Only one thread runs uninterrupted after panic if stop_scheduler_on_panic
is set. That thread might call code that is also used in normal context
and that code might use locks to prevent concurrent execution of certain
parts. Those locks might be held by the stopped threads and would never
be released. To work around this issue, it was decided that instead of
explicit checks for panic context, we would rather put those checks
inside the locking primitives.
This change has substantial portions written and re-written by attilio
and kib at various times. Other changes are heavily based on the ideas
and patches submitted by jhb and mdf. bde has provided many insights
into the details and history of the current code.
The new behavior may cause problems for systems that use a USB keyboard
for interfacing with system console. This is because of some unusual
locking patterns in the ukbd code which have to be used because on one
hand ukbd is below syscons, but on the other hand it has to interface
with other usb code that uses regular mutexes/Giant for its concurrency
protection. Dumping to USB-connected disks may also be affected.
PR: amd64/139614 (at least)
In cooperation with: attilio, jhb, kib, mdf
Discussed with: arch@, bde
Tested by: Eugene Grosbein <eugen@grosbein.net>,
gnn,
Steven Hartland <killing@multiplay.co.uk>,
glebius,
Andrew Boyer <aboyer@averesystems.com>
(various versions of the patch)
MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
|
|
|
if (SCHEDULER_STOPPED())
|
|
|
|
return;
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
curthread->td_locks--;
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
WITNESS_UNLOCK(&rm->rm_lock_sx.lock_object, LOP_EXCLUSIVE,
|
|
|
|
file, line);
|
|
|
|
else
|
|
|
|
WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
|
2007-11-08 14:47:55 +00:00
|
|
|
LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line);
|
|
|
|
_rm_wunlock(rm);
|
2009-04-26 21:16:03 +00:00
|
|
|
}
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
int
|
2007-11-08 14:47:55 +00:00
|
|
|
_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
|
2010-09-01 19:50:03 +00:00
|
|
|
int trylock, const char *file, int line)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
panic: add a switch and infrastructure for stopping other CPUs in SMP case
Historical behavior of letting other CPUs merily go on is a default for
time being. The new behavior can be switched on via
kern.stop_scheduler_on_panic tunable and sysctl.
Stopping of the CPUs has (at least) the following benefits:
- more of the system state at panic time is preserved intact
- threads and interrupts do not interfere with dumping of the system
state
Only one thread runs uninterrupted after panic if stop_scheduler_on_panic
is set. That thread might call code that is also used in normal context
and that code might use locks to prevent concurrent execution of certain
parts. Those locks might be held by the stopped threads and would never
be released. To work around this issue, it was decided that instead of
explicit checks for panic context, we would rather put those checks
inside the locking primitives.
This change has substantial portions written and re-written by attilio
and kib at various times. Other changes are heavily based on the ideas
and patches submitted by jhb and mdf. bde has provided many insights
into the details and history of the current code.
The new behavior may cause problems for systems that use a USB keyboard
for interfacing with system console. This is because of some unusual
locking patterns in the ukbd code which have to be used because on one
hand ukbd is below syscons, but on the other hand it has to interface
with other usb code that uses regular mutexes/Giant for its concurrency
protection. Dumping to USB-connected disks may also be affected.
PR: amd64/139614 (at least)
In cooperation with: attilio, jhb, kib, mdf
Discussed with: arch@, bde
Tested by: Eugene Grosbein <eugen@grosbein.net>,
gnn,
Steven Hartland <killing@multiplay.co.uk>,
glebius,
Andrew Boyer <aboyer@averesystems.com>
(various versions of the patch)
MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
|
|
|
|
|
|
|
if (SCHEDULER_STOPPED())
|
|
|
|
return (1);
|
|
|
|
|
2012-09-12 22:10:53 +00:00
|
|
|
KASSERT(!TD_IS_IDLETHREAD(curthread),
|
|
|
|
("rm_rlock() by idle thread %p on rmlock %s @ %s:%d",
|
|
|
|
curthread, rm->lock_object.lo_name, file, line));
|
2010-09-01 19:50:03 +00:00
|
|
|
if (!trylock && (rm->lock_object.lo_flags & RM_SLEEPABLE))
|
|
|
|
WITNESS_CHECKORDER(&rm->rm_lock_sx.lock_object, LOP_NEWORDER,
|
|
|
|
file, line, NULL);
|
2009-04-26 21:16:03 +00:00
|
|
|
WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line, NULL);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (_rm_rlock(rm, tracker, trylock)) {
|
|
|
|
LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file, line);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
WITNESS_LOCK(&rm->lock_object, 0, file, line);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
curthread->td_locks++;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
return (1);
|
|
|
|
}
|
|
|
|
|
|
|
|
return (0);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
void
|
2009-12-25 21:14:05 +00:00
|
|
|
_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
|
2009-04-26 21:16:03 +00:00
|
|
|
const char *file, int line)
|
|
|
|
{
|
|
|
|
|
panic: add a switch and infrastructure for stopping other CPUs in SMP case
Historical behavior of letting other CPUs merily go on is a default for
time being. The new behavior can be switched on via
kern.stop_scheduler_on_panic tunable and sysctl.
Stopping of the CPUs has (at least) the following benefits:
- more of the system state at panic time is preserved intact
- threads and interrupts do not interfere with dumping of the system
state
Only one thread runs uninterrupted after panic if stop_scheduler_on_panic
is set. That thread might call code that is also used in normal context
and that code might use locks to prevent concurrent execution of certain
parts. Those locks might be held by the stopped threads and would never
be released. To work around this issue, it was decided that instead of
explicit checks for panic context, we would rather put those checks
inside the locking primitives.
This change has substantial portions written and re-written by attilio
and kib at various times. Other changes are heavily based on the ideas
and patches submitted by jhb and mdf. bde has provided many insights
into the details and history of the current code.
The new behavior may cause problems for systems that use a USB keyboard
for interfacing with system console. This is because of some unusual
locking patterns in the ukbd code which have to be used because on one
hand ukbd is below syscons, but on the other hand it has to interface
with other usb code that uses regular mutexes/Giant for its concurrency
protection. Dumping to USB-connected disks may also be affected.
PR: amd64/139614 (at least)
In cooperation with: attilio, jhb, kib, mdf
Discussed with: arch@, bde
Tested by: Eugene Grosbein <eugen@grosbein.net>,
gnn,
Steven Hartland <killing@multiplay.co.uk>,
glebius,
Andrew Boyer <aboyer@averesystems.com>
(various versions of the patch)
MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
|
|
|
if (SCHEDULER_STOPPED())
|
|
|
|
return;
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
curthread->td_locks--;
|
2009-04-26 21:16:03 +00:00
|
|
|
WITNESS_UNLOCK(&rm->lock_object, 0, file, line);
|
2007-11-08 14:47:55 +00:00
|
|
|
LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line);
|
|
|
|
_rm_runlock(rm, tracker);
|
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/*
|
|
|
|
* Just strip out file and line arguments if no lock debugging is enabled in
|
|
|
|
* the kernel - we are called from a kernel module.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
_rm_wlock_debug(struct rmlock *rm, const char *file, int line)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
_rm_wlock(rm);
|
|
|
|
}
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
void
|
|
|
|
_rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
_rm_wunlock(rm);
|
2009-04-26 21:16:03 +00:00
|
|
|
}
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
int
|
2007-11-08 14:47:55 +00:00
|
|
|
_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
|
2010-09-01 19:50:03 +00:00
|
|
|
int trylock, const char *file, int line)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
return _rm_rlock(rm, tracker, trylock);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
void
|
2009-12-25 21:14:05 +00:00
|
|
|
_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
|
2009-05-29 14:25:51 +00:00
|
|
|
const char *file, int line)
|
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
_rm_runlock(rm, tracker);
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|