freebsd-dev/sys/kern/kern_rmlock.c

861 lines
20 KiB
C
Raw Normal View History

/*-
* Copyright (c) 2007 Stephan Uphoff <ups@FreeBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the author nor the names of any co-contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* Machine independent bits of reader/writer lock implementation.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "opt_ddb.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/kdb.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/rmlock.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/turnstile.h>
#include <sys/lock_profile.h>
#include <machine/cpu.h>
#ifdef DDB
#include <ddb/ddb.h>
#endif
2009-04-26 21:16:03 +00:00
/*
* A cookie to mark destroyed rmlocks. This is stored in the head of
* rm_activeReaders.
*/
#define RM_DESTROYED ((void *)0xdead)
#define rm_destroyed(rm) \
(LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED)
#define RMPF_ONQUEUE 1
#define RMPF_SIGNAL 2
#ifndef INVARIANTS
#define _rm_assert(c, what, file, line)
#endif
static void assert_rm(const struct lock_object *lock, int what);
#ifdef DDB
static void db_show_rm(const struct lock_object *lock);
#endif
static void lock_rm(struct lock_object *lock, uintptr_t how);
#ifdef KDTRACE_HOOKS
static int owner_rm(const struct lock_object *lock, struct thread **owner);
#endif
static uintptr_t unlock_rm(struct lock_object *lock);
struct lock_class lock_class_rm = {
.lc_name = "rm",
.lc_flags = LC_SLEEPLOCK | LC_RECURSABLE,
.lc_assert = assert_rm,
#ifdef DDB
.lc_ddb_show = db_show_rm,
#endif
.lc_lock = lock_rm,
.lc_unlock = unlock_rm,
#ifdef KDTRACE_HOOKS
.lc_owner = owner_rm,
#endif
};
struct lock_class lock_class_rm_sleepable = {
.lc_name = "sleepable rm",
.lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE,
.lc_assert = assert_rm,
#ifdef DDB
.lc_ddb_show = db_show_rm,
#endif
.lc_lock = lock_rm,
.lc_unlock = unlock_rm,
#ifdef KDTRACE_HOOKS
.lc_owner = owner_rm,
#endif
};
static void
assert_rm(const struct lock_object *lock, int what)
{
rm_assert((const struct rmlock *)lock, what);
}
static void
lock_rm(struct lock_object *lock, uintptr_t how)
2009-04-26 21:16:03 +00:00
{
struct rmlock *rm;
struct rm_priotracker *tracker;
2009-04-26 21:16:03 +00:00
rm = (struct rmlock *)lock;
if (how == 0)
rm_wlock(rm);
else {
tracker = (struct rm_priotracker *)how;
rm_rlock(rm, tracker);
}
}
static uintptr_t
2009-04-26 21:16:03 +00:00
unlock_rm(struct lock_object *lock)
{
struct thread *td;
struct pcpu *pc;
struct rmlock *rm;
struct rm_queue *queue;
struct rm_priotracker *tracker;
uintptr_t how;
2009-04-26 21:16:03 +00:00
rm = (struct rmlock *)lock;
tracker = NULL;
how = 0;
rm_assert(rm, RA_LOCKED | RA_NOTRECURSED);
if (rm_wowned(rm))
rm_wunlock(rm);
else {
/*
* Find the right rm_priotracker structure for curthread.
* The guarantee about its uniqueness is given by the fact
* we already asserted the lock wasn't recursively acquired.
*/
critical_enter();
td = curthread;
pc = pcpu_find(curcpu);
for (queue = pc->pc_rm_queue.rmq_next;
queue != &pc->pc_rm_queue; queue = queue->rmq_next) {
tracker = (struct rm_priotracker *)queue;
if ((tracker->rmp_rmlock == rm) &&
(tracker->rmp_thread == td)) {
how = (uintptr_t)tracker;
break;
}
}
KASSERT(tracker != NULL,
("rm_priotracker is non-NULL when lock held in read mode"));
critical_exit();
rm_runlock(rm, tracker);
}
return (how);
}
#ifdef KDTRACE_HOOKS
static int
owner_rm(const struct lock_object *lock, struct thread **owner)
{
const struct rmlock *rm;
struct lock_class *lc;
rm = (const struct rmlock *)lock;
lc = LOCK_CLASS(&rm->rm_wlock_object);
return (lc->lc_owner(&rm->rm_wlock_object, owner));
}
#endif
static struct mtx rm_spinlock;
MTX_SYSINIT(rm_spinlock, &rm_spinlock, "rm_spinlock", MTX_SPIN);
/*
* Add or remove tracker from per-cpu list.
2009-04-26 21:16:03 +00:00
*
* The per-cpu list can be traversed at any time in forward direction from an
2009-04-26 21:16:03 +00:00
* interrupt on the *local* cpu.
*/
2009-04-26 21:16:03 +00:00
static void inline
rm_tracker_add(struct pcpu *pc, struct rm_priotracker *tracker)
{
struct rm_queue *next;
/* Initialize all tracker pointers */
tracker->rmp_cpuQueue.rmq_prev = &pc->pc_rm_queue;
next = pc->pc_rm_queue.rmq_next;
tracker->rmp_cpuQueue.rmq_next = next;
2009-04-26 21:16:03 +00:00
/* rmq_prev is not used during froward traversal. */
next->rmq_prev = &tracker->rmp_cpuQueue;
2009-04-26 21:16:03 +00:00
/* Update pointer to first element. */
pc->pc_rm_queue.rmq_next = &tracker->rmp_cpuQueue;
}
/*
* Return a count of the number of trackers the thread 'td' already
* has on this CPU for the lock 'rm'.
*/
static int
rm_trackers_present(const struct pcpu *pc, const struct rmlock *rm,
const struct thread *td)
{
struct rm_queue *queue;
struct rm_priotracker *tracker;
int count;
count = 0;
for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue;
queue = queue->rmq_next) {
tracker = (struct rm_priotracker *)queue;
if ((tracker->rmp_rmlock == rm) && (tracker->rmp_thread == td))
count++;
}
return (count);
}
2009-04-26 21:16:03 +00:00
static void inline
rm_tracker_remove(struct pcpu *pc, struct rm_priotracker *tracker)
{
struct rm_queue *next, *prev;
2009-04-26 21:16:03 +00:00
next = tracker->rmp_cpuQueue.rmq_next;
prev = tracker->rmp_cpuQueue.rmq_prev;
2009-04-26 21:16:03 +00:00
/* Not used during forward traversal. */
next->rmq_prev = prev;
2009-04-26 21:16:03 +00:00
/* Remove from list. */
prev->rmq_next = next;
}
2009-04-26 21:16:03 +00:00
static void
rm_cleanIPI(void *arg)
{
struct pcpu *pc;
2009-04-26 21:16:03 +00:00
struct rmlock *rm = arg;
struct rm_priotracker *tracker;
2009-04-26 21:16:03 +00:00
struct rm_queue *queue;
pc = pcpu_find(curcpu);
2009-04-26 21:16:03 +00:00
for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue;
queue = queue->rmq_next) {
2009-04-26 21:16:03 +00:00
tracker = (struct rm_priotracker *)queue;
if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) {
tracker->rmp_flags = RMPF_ONQUEUE;
mtx_lock_spin(&rm_spinlock);
2009-04-26 21:16:03 +00:00
LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
rmp_qentry);
mtx_unlock_spin(&rm_spinlock);
}
}
}
void
rm_init_flags(struct rmlock *rm, const char *name, int opts)
{
struct lock_class *lc;
int liflags, xflags;
2009-04-26 21:16:03 +00:00
liflags = 0;
if (!(opts & RM_NOWITNESS))
liflags |= LO_WITNESS;
if (opts & RM_RECURSE)
liflags |= LO_RECURSABLE;
if (opts & RM_NEW)
liflags |= LO_NEW;
rm->rm_writecpus = all_cpus;
LIST_INIT(&rm->rm_activeReaders);
if (opts & RM_SLEEPABLE) {
liflags |= LO_SLEEPABLE;
lc = &lock_class_rm_sleepable;
xflags = (opts & RM_NEW ? SX_NEW : 0);
sx_init_flags(&rm->rm_lock_sx, "rmlock_sx",
xflags | SX_NOWITNESS);
} else {
lc = &lock_class_rm;
xflags = (opts & RM_NEW ? MTX_NEW : 0);
mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx",
xflags | MTX_NOWITNESS);
}
lock_init(&rm->lock_object, lc, name, NULL, liflags);
}
void
rm_init(struct rmlock *rm, const char *name)
{
rm_init_flags(rm, name, 0);
}
void
rm_destroy(struct rmlock *rm)
{
2009-04-26 21:16:03 +00:00
rm_assert(rm, RA_UNLOCKED);
LIST_FIRST(&rm->rm_activeReaders) = RM_DESTROYED;
if (rm->lock_object.lo_flags & LO_SLEEPABLE)
sx_destroy(&rm->rm_lock_sx);
else
mtx_destroy(&rm->rm_lock_mtx);
lock_destroy(&rm->lock_object);
}
int
rm_wowned(const struct rmlock *rm)
{
if (rm->lock_object.lo_flags & LO_SLEEPABLE)
return (sx_xlocked(&rm->rm_lock_sx));
else
return (mtx_owned(&rm->rm_lock_mtx));
}
void
rm_sysinit(void *arg)
{
struct rm_args *args = arg;
rm_init(args->ra_rm, args->ra_desc);
}
void
rm_sysinit_flags(void *arg)
{
struct rm_args_flags *args = arg;
rm_init_flags(args->ra_rm, args->ra_desc, args->ra_opts);
}
static int
_rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
{
struct pcpu *pc;
critical_enter();
pc = pcpu_find(curcpu);
2009-04-26 21:16:03 +00:00
/* Check if we just need to do a proper critical_exit. */
if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) {
critical_exit();
return (1);
}
/* Remove our tracker from the per-cpu list. */
2009-04-26 21:16:03 +00:00
rm_tracker_remove(pc, tracker);
2009-04-26 21:16:03 +00:00
/* Check to see if the IPI granted us the lock after all. */
if (tracker->rmp_flags) {
/* Just add back tracker - we hold the lock. */
rm_tracker_add(pc, tracker);
critical_exit();
return (1);
}
/*
2009-04-26 21:16:03 +00:00
* We allow readers to aquire a lock even if a writer is blocked if
* the lock is recursive and the reader already holds the lock.
*/
if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) {
/*
* Just grant the lock if this thread already has a tracker
* for this lock on the per-cpu queue.
*/
if (rm_trackers_present(pc, rm, curthread) != 0) {
mtx_lock_spin(&rm_spinlock);
LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
rmp_qentry);
tracker->rmp_flags = RMPF_ONQUEUE;
mtx_unlock_spin(&rm_spinlock);
rm_tracker_add(pc, tracker);
critical_exit();
return (1);
}
}
sched_unpin();
critical_exit();
if (trylock) {
if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
if (!sx_try_xlock(&rm->rm_lock_sx))
return (0);
} else {
if (!mtx_trylock(&rm->rm_lock_mtx))
return (0);
}
} else {
if (rm->lock_object.lo_flags & LO_SLEEPABLE) {
THREAD_SLEEPING_OK();
sx_xlock(&rm->rm_lock_sx);
THREAD_NO_SLEEPING();
} else
mtx_lock(&rm->rm_lock_mtx);
}
critical_enter();
pc = pcpu_find(curcpu);
CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus);
2009-04-26 21:16:03 +00:00
rm_tracker_add(pc, tracker);
sched_pin();
critical_exit();
2009-04-26 21:16:03 +00:00
if (rm->lock_object.lo_flags & LO_SLEEPABLE)
sx_xunlock(&rm->rm_lock_sx);
else
mtx_unlock(&rm->rm_lock_mtx);
return (1);
}
int
_rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
{
struct thread *td = curthread;
struct pcpu *pc;
panic: add a switch and infrastructure for stopping other CPUs in SMP case Historical behavior of letting other CPUs merily go on is a default for time being. The new behavior can be switched on via kern.stop_scheduler_on_panic tunable and sysctl. Stopping of the CPUs has (at least) the following benefits: - more of the system state at panic time is preserved intact - threads and interrupts do not interfere with dumping of the system state Only one thread runs uninterrupted after panic if stop_scheduler_on_panic is set. That thread might call code that is also used in normal context and that code might use locks to prevent concurrent execution of certain parts. Those locks might be held by the stopped threads and would never be released. To work around this issue, it was decided that instead of explicit checks for panic context, we would rather put those checks inside the locking primitives. This change has substantial portions written and re-written by attilio and kib at various times. Other changes are heavily based on the ideas and patches submitted by jhb and mdf. bde has provided many insights into the details and history of the current code. The new behavior may cause problems for systems that use a USB keyboard for interfacing with system console. This is because of some unusual locking patterns in the ukbd code which have to be used because on one hand ukbd is below syscons, but on the other hand it has to interface with other usb code that uses regular mutexes/Giant for its concurrency protection. Dumping to USB-connected disks may also be affected. PR: amd64/139614 (at least) In cooperation with: attilio, jhb, kib, mdf Discussed with: arch@, bde Tested by: Eugene Grosbein <eugen@grosbein.net>, gnn, Steven Hartland <killing@multiplay.co.uk>, glebius, Andrew Boyer <aboyer@averesystems.com> (various versions of the patch) MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
if (SCHEDULER_STOPPED())
return (1);
tracker->rmp_flags = 0;
tracker->rmp_thread = td;
tracker->rmp_rmlock = rm;
if (rm->lock_object.lo_flags & LO_SLEEPABLE)
THREAD_NO_SLEEPING();
td->td_critnest++; /* critical_enter(); */
__compiler_membar();
pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */
2009-04-26 21:16:03 +00:00
rm_tracker_add(pc, tracker);
sched_pin();
__compiler_membar();
td->td_critnest--;
2009-04-26 21:16:03 +00:00
/*
* Fast path to combine two common conditions into a single
* conditional jump.
*/
Commit the support for removing cpumask_t and replacing it directly with cpuset_t objects. That is going to offer the underlying support for a simple bump of MAXCPU and then support for number of cpus > 32 (as it is today). Right now, cpumask_t is an int, 32 bits on all our supported architecture. cpumask_t on the other side is implemented as an array of longs, and easilly extendible by definition. The architectures touched by this commit are the following: - amd64 - i386 - pc98 - arm - ia64 - XEN while the others are still missing. Userland is believed to be fully converted with the changes contained here. Some technical notes: - This commit may be considered an ABI nop for all the architectures different from amd64 and ia64 (and sparc64 in the future) - per-cpu members, which are now converted to cpuset_t, needs to be accessed avoiding migration, because the size of cpuset_t should be considered unknown - size of cpuset_t objects is different from kernel and userland (this is primirally done in order to leave some more space in userland to cope with KBI extensions). If you need to access kernel cpuset_t from the userland please refer to example in this patch on how to do that correctly (kgdb may be a good source, for example). - Support for other architectures is going to be added soon - Only MAXCPU for amd64 is bumped now The patch has been tested by sbruno and Nicholas Esborn on opteron 4 x 12 pack CPUs. More testing on big SMP is expected to came soon. pluknet tested the patch with his 8-ways on both amd64 and i386. Tested by: pluknet, sbruno, gianni, Nicholas Esborn Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
if (0 == (td->td_owepreempt |
CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)))
return (1);
2009-04-26 21:16:03 +00:00
/* We do not have a read token and need to acquire one. */
return _rm_rlock_hard(rm, tracker, trylock);
}
static void
2009-04-26 21:16:03 +00:00
_rm_unlock_hard(struct thread *td,struct rm_priotracker *tracker)
{
2009-04-26 21:16:03 +00:00
if (td->td_owepreempt) {
td->td_critnest++;
critical_exit();
}
2009-04-26 21:16:03 +00:00
if (!tracker->rmp_flags)
return;
mtx_lock_spin(&rm_spinlock);
2009-04-26 21:16:03 +00:00
LIST_REMOVE(tracker, rmp_qentry);
if (tracker->rmp_flags & RMPF_SIGNAL) {
struct rmlock *rm;
2009-04-26 21:16:03 +00:00
struct turnstile *ts;
rm = tracker->rmp_rmlock;
2009-04-26 21:16:03 +00:00
turnstile_chain_lock(&rm->lock_object);
mtx_unlock_spin(&rm_spinlock);
ts = turnstile_lookup(&rm->lock_object);
turnstile_signal(ts, TS_EXCLUSIVE_QUEUE);
turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
turnstile_chain_unlock(&rm->lock_object);
} else
mtx_unlock_spin(&rm_spinlock);
2009-04-26 21:16:03 +00:00
}
void
2009-04-26 21:16:03 +00:00
_rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker)
{
struct pcpu *pc;
struct thread *td = tracker->rmp_thread;
panic: add a switch and infrastructure for stopping other CPUs in SMP case Historical behavior of letting other CPUs merily go on is a default for time being. The new behavior can be switched on via kern.stop_scheduler_on_panic tunable and sysctl. Stopping of the CPUs has (at least) the following benefits: - more of the system state at panic time is preserved intact - threads and interrupts do not interfere with dumping of the system state Only one thread runs uninterrupted after panic if stop_scheduler_on_panic is set. That thread might call code that is also used in normal context and that code might use locks to prevent concurrent execution of certain parts. Those locks might be held by the stopped threads and would never be released. To work around this issue, it was decided that instead of explicit checks for panic context, we would rather put those checks inside the locking primitives. This change has substantial portions written and re-written by attilio and kib at various times. Other changes are heavily based on the ideas and patches submitted by jhb and mdf. bde has provided many insights into the details and history of the current code. The new behavior may cause problems for systems that use a USB keyboard for interfacing with system console. This is because of some unusual locking patterns in the ukbd code which have to be used because on one hand ukbd is below syscons, but on the other hand it has to interface with other usb code that uses regular mutexes/Giant for its concurrency protection. Dumping to USB-connected disks may also be affected. PR: amd64/139614 (at least) In cooperation with: attilio, jhb, kib, mdf Discussed with: arch@, bde Tested by: Eugene Grosbein <eugen@grosbein.net>, gnn, Steven Hartland <killing@multiplay.co.uk>, glebius, Andrew Boyer <aboyer@averesystems.com> (various versions of the patch) MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
if (SCHEDULER_STOPPED())
return;
td->td_critnest++; /* critical_enter(); */
pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */
2009-04-26 21:16:03 +00:00
rm_tracker_remove(pc, tracker);
td->td_critnest--;
sched_unpin();
if (rm->lock_object.lo_flags & LO_SLEEPABLE)
THREAD_SLEEPING_OK();
2009-04-26 21:16:03 +00:00
if (0 == (td->td_owepreempt | tracker->rmp_flags))
return;
2009-04-26 21:16:03 +00:00
_rm_unlock_hard(td, tracker);
}
void
_rm_wlock(struct rmlock *rm)
{
struct rm_priotracker *prio;
struct turnstile *ts;
Commit the support for removing cpumask_t and replacing it directly with cpuset_t objects. That is going to offer the underlying support for a simple bump of MAXCPU and then support for number of cpus > 32 (as it is today). Right now, cpumask_t is an int, 32 bits on all our supported architecture. cpumask_t on the other side is implemented as an array of longs, and easilly extendible by definition. The architectures touched by this commit are the following: - amd64 - i386 - pc98 - arm - ia64 - XEN while the others are still missing. Userland is believed to be fully converted with the changes contained here. Some technical notes: - This commit may be considered an ABI nop for all the architectures different from amd64 and ia64 (and sparc64 in the future) - per-cpu members, which are now converted to cpuset_t, needs to be accessed avoiding migration, because the size of cpuset_t should be considered unknown - size of cpuset_t objects is different from kernel and userland (this is primirally done in order to leave some more space in userland to cope with KBI extensions). If you need to access kernel cpuset_t from the userland please refer to example in this patch on how to do that correctly (kgdb may be a good source, for example). - Support for other architectures is going to be added soon - Only MAXCPU for amd64 is bumped now The patch has been tested by sbruno and Nicholas Esborn on opteron 4 x 12 pack CPUs. More testing on big SMP is expected to came soon. pluknet tested the patch with his 8-ways on both amd64 and i386. Tested by: pluknet, sbruno, gianni, Nicholas Esborn Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
cpuset_t readcpus;
panic: add a switch and infrastructure for stopping other CPUs in SMP case Historical behavior of letting other CPUs merily go on is a default for time being. The new behavior can be switched on via kern.stop_scheduler_on_panic tunable and sysctl. Stopping of the CPUs has (at least) the following benefits: - more of the system state at panic time is preserved intact - threads and interrupts do not interfere with dumping of the system state Only one thread runs uninterrupted after panic if stop_scheduler_on_panic is set. That thread might call code that is also used in normal context and that code might use locks to prevent concurrent execution of certain parts. Those locks might be held by the stopped threads and would never be released. To work around this issue, it was decided that instead of explicit checks for panic context, we would rather put those checks inside the locking primitives. This change has substantial portions written and re-written by attilio and kib at various times. Other changes are heavily based on the ideas and patches submitted by jhb and mdf. bde has provided many insights into the details and history of the current code. The new behavior may cause problems for systems that use a USB keyboard for interfacing with system console. This is because of some unusual locking patterns in the ukbd code which have to be used because on one hand ukbd is below syscons, but on the other hand it has to interface with other usb code that uses regular mutexes/Giant for its concurrency protection. Dumping to USB-connected disks may also be affected. PR: amd64/139614 (at least) In cooperation with: attilio, jhb, kib, mdf Discussed with: arch@, bde Tested by: Eugene Grosbein <eugen@grosbein.net>, gnn, Steven Hartland <killing@multiplay.co.uk>, glebius, Andrew Boyer <aboyer@averesystems.com> (various versions of the patch) MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
if (SCHEDULER_STOPPED())
return;
if (rm->lock_object.lo_flags & LO_SLEEPABLE)
sx_xlock(&rm->rm_lock_sx);
else
mtx_lock(&rm->rm_lock_mtx);
Commit the support for removing cpumask_t and replacing it directly with cpuset_t objects. That is going to offer the underlying support for a simple bump of MAXCPU and then support for number of cpus > 32 (as it is today). Right now, cpumask_t is an int, 32 bits on all our supported architecture. cpumask_t on the other side is implemented as an array of longs, and easilly extendible by definition. The architectures touched by this commit are the following: - amd64 - i386 - pc98 - arm - ia64 - XEN while the others are still missing. Userland is believed to be fully converted with the changes contained here. Some technical notes: - This commit may be considered an ABI nop for all the architectures different from amd64 and ia64 (and sparc64 in the future) - per-cpu members, which are now converted to cpuset_t, needs to be accessed avoiding migration, because the size of cpuset_t should be considered unknown - size of cpuset_t objects is different from kernel and userland (this is primirally done in order to leave some more space in userland to cope with KBI extensions). If you need to access kernel cpuset_t from the userland please refer to example in this patch on how to do that correctly (kgdb may be a good source, for example). - Support for other architectures is going to be added soon - Only MAXCPU for amd64 is bumped now The patch has been tested by sbruno and Nicholas Esborn on opteron 4 x 12 pack CPUs. More testing on big SMP is expected to came soon. pluknet tested the patch with his 8-ways on both amd64 and i386. Tested by: pluknet, sbruno, gianni, Nicholas Esborn Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) {
/* Get all read tokens back */
Commit the support for removing cpumask_t and replacing it directly with cpuset_t objects. That is going to offer the underlying support for a simple bump of MAXCPU and then support for number of cpus > 32 (as it is today). Right now, cpumask_t is an int, 32 bits on all our supported architecture. cpumask_t on the other side is implemented as an array of longs, and easilly extendible by definition. The architectures touched by this commit are the following: - amd64 - i386 - pc98 - arm - ia64 - XEN while the others are still missing. Userland is believed to be fully converted with the changes contained here. Some technical notes: - This commit may be considered an ABI nop for all the architectures different from amd64 and ia64 (and sparc64 in the future) - per-cpu members, which are now converted to cpuset_t, needs to be accessed avoiding migration, because the size of cpuset_t should be considered unknown - size of cpuset_t objects is different from kernel and userland (this is primirally done in order to leave some more space in userland to cope with KBI extensions). If you need to access kernel cpuset_t from the userland please refer to example in this patch on how to do that correctly (kgdb may be a good source, for example). - Support for other architectures is going to be added soon - Only MAXCPU for amd64 is bumped now The patch has been tested by sbruno and Nicholas Esborn on opteron 4 x 12 pack CPUs. More testing on big SMP is expected to came soon. pluknet tested the patch with his 8-ways on both amd64 and i386. Tested by: pluknet, sbruno, gianni, Nicholas Esborn Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
readcpus = all_cpus;
CPU_NAND(&readcpus, &rm->rm_writecpus);
rm->rm_writecpus = all_cpus;
2009-04-26 21:16:03 +00:00
/*
* Assumes rm->rm_writecpus update is visible on other CPUs
2009-04-26 21:16:03 +00:00
* before rm_cleanIPI is called.
*/
#ifdef SMP
smp_rendezvous_cpus(readcpus,
smp_no_rendevous_barrier,
rm_cleanIPI,
2009-04-26 21:16:03 +00:00
smp_no_rendevous_barrier,
rm);
#else
rm_cleanIPI(rm);
#endif
mtx_lock_spin(&rm_spinlock);
2009-04-26 21:16:03 +00:00
while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) {
ts = turnstile_trywait(&rm->lock_object);
prio->rmp_flags = RMPF_ONQUEUE | RMPF_SIGNAL;
mtx_unlock_spin(&rm_spinlock);
2009-04-26 21:16:03 +00:00
turnstile_wait(ts, prio->rmp_thread,
TS_EXCLUSIVE_QUEUE);
mtx_lock_spin(&rm_spinlock);
}
mtx_unlock_spin(&rm_spinlock);
}
}
void
_rm_wunlock(struct rmlock *rm)
{
2009-04-26 21:16:03 +00:00
if (rm->lock_object.lo_flags & LO_SLEEPABLE)
sx_xunlock(&rm->rm_lock_sx);
else
mtx_unlock(&rm->rm_lock_mtx);
}
#if LOCK_DEBUG > 0
void
_rm_wlock_debug(struct rmlock *rm, const char *file, int line)
{
panic: add a switch and infrastructure for stopping other CPUs in SMP case Historical behavior of letting other CPUs merily go on is a default for time being. The new behavior can be switched on via kern.stop_scheduler_on_panic tunable and sysctl. Stopping of the CPUs has (at least) the following benefits: - more of the system state at panic time is preserved intact - threads and interrupts do not interfere with dumping of the system state Only one thread runs uninterrupted after panic if stop_scheduler_on_panic is set. That thread might call code that is also used in normal context and that code might use locks to prevent concurrent execution of certain parts. Those locks might be held by the stopped threads and would never be released. To work around this issue, it was decided that instead of explicit checks for panic context, we would rather put those checks inside the locking primitives. This change has substantial portions written and re-written by attilio and kib at various times. Other changes are heavily based on the ideas and patches submitted by jhb and mdf. bde has provided many insights into the details and history of the current code. The new behavior may cause problems for systems that use a USB keyboard for interfacing with system console. This is because of some unusual locking patterns in the ukbd code which have to be used because on one hand ukbd is below syscons, but on the other hand it has to interface with other usb code that uses regular mutexes/Giant for its concurrency protection. Dumping to USB-connected disks may also be affected. PR: amd64/139614 (at least) In cooperation with: attilio, jhb, kib, mdf Discussed with: arch@, bde Tested by: Eugene Grosbein <eugen@grosbein.net>, gnn, Steven Hartland <killing@multiplay.co.uk>, glebius, Andrew Boyer <aboyer@averesystems.com> (various versions of the patch) MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
if (SCHEDULER_STOPPED())
return;
KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
("rm_wlock() by idle thread %p on rmlock %s @ %s:%d",
curthread, rm->lock_object.lo_name, file, line));
KASSERT(!rm_destroyed(rm),
("rm_wlock() of destroyed rmlock @ %s:%d", file, line));
_rm_assert(rm, RA_UNLOCKED, file, line);
2009-04-26 21:16:03 +00:00
WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE,
file, line, NULL);
_rm_wlock(rm);
LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line);
WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
TD_LOCKS_INC(curthread);
}
2009-04-26 21:16:03 +00:00
void
_rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
{
2009-04-26 21:16:03 +00:00
panic: add a switch and infrastructure for stopping other CPUs in SMP case Historical behavior of letting other CPUs merily go on is a default for time being. The new behavior can be switched on via kern.stop_scheduler_on_panic tunable and sysctl. Stopping of the CPUs has (at least) the following benefits: - more of the system state at panic time is preserved intact - threads and interrupts do not interfere with dumping of the system state Only one thread runs uninterrupted after panic if stop_scheduler_on_panic is set. That thread might call code that is also used in normal context and that code might use locks to prevent concurrent execution of certain parts. Those locks might be held by the stopped threads and would never be released. To work around this issue, it was decided that instead of explicit checks for panic context, we would rather put those checks inside the locking primitives. This change has substantial portions written and re-written by attilio and kib at various times. Other changes are heavily based on the ideas and patches submitted by jhb and mdf. bde has provided many insights into the details and history of the current code. The new behavior may cause problems for systems that use a USB keyboard for interfacing with system console. This is because of some unusual locking patterns in the ukbd code which have to be used because on one hand ukbd is below syscons, but on the other hand it has to interface with other usb code that uses regular mutexes/Giant for its concurrency protection. Dumping to USB-connected disks may also be affected. PR: amd64/139614 (at least) In cooperation with: attilio, jhb, kib, mdf Discussed with: arch@, bde Tested by: Eugene Grosbein <eugen@grosbein.net>, gnn, Steven Hartland <killing@multiplay.co.uk>, glebius, Andrew Boyer <aboyer@averesystems.com> (various versions of the patch) MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
if (SCHEDULER_STOPPED())
return;
KASSERT(!rm_destroyed(rm),
("rm_wunlock() of destroyed rmlock @ %s:%d", file, line));
_rm_assert(rm, RA_WLOCKED, file, line);
WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line);
_rm_wunlock(rm);
TD_LOCKS_DEC(curthread);
2009-04-26 21:16:03 +00:00
}
int
_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
int trylock, const char *file, int line)
{
panic: add a switch and infrastructure for stopping other CPUs in SMP case Historical behavior of letting other CPUs merily go on is a default for time being. The new behavior can be switched on via kern.stop_scheduler_on_panic tunable and sysctl. Stopping of the CPUs has (at least) the following benefits: - more of the system state at panic time is preserved intact - threads and interrupts do not interfere with dumping of the system state Only one thread runs uninterrupted after panic if stop_scheduler_on_panic is set. That thread might call code that is also used in normal context and that code might use locks to prevent concurrent execution of certain parts. Those locks might be held by the stopped threads and would never be released. To work around this issue, it was decided that instead of explicit checks for panic context, we would rather put those checks inside the locking primitives. This change has substantial portions written and re-written by attilio and kib at various times. Other changes are heavily based on the ideas and patches submitted by jhb and mdf. bde has provided many insights into the details and history of the current code. The new behavior may cause problems for systems that use a USB keyboard for interfacing with system console. This is because of some unusual locking patterns in the ukbd code which have to be used because on one hand ukbd is below syscons, but on the other hand it has to interface with other usb code that uses regular mutexes/Giant for its concurrency protection. Dumping to USB-connected disks may also be affected. PR: amd64/139614 (at least) In cooperation with: attilio, jhb, kib, mdf Discussed with: arch@, bde Tested by: Eugene Grosbein <eugen@grosbein.net>, gnn, Steven Hartland <killing@multiplay.co.uk>, glebius, Andrew Boyer <aboyer@averesystems.com> (various versions of the patch) MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
if (SCHEDULER_STOPPED())
return (1);
#ifdef INVARIANTS
if (!(rm->lock_object.lo_flags & LO_RECURSABLE) && !trylock) {
critical_enter();
KASSERT(rm_trackers_present(pcpu_find(curcpu), rm,
curthread) == 0,
("rm_rlock: recursed on non-recursive rmlock %s @ %s:%d\n",
rm->lock_object.lo_name, file, line));
critical_exit();
}
#endif
KASSERT(kdb_active != 0 || !TD_IS_IDLETHREAD(curthread),
("rm_rlock() by idle thread %p on rmlock %s @ %s:%d",
curthread, rm->lock_object.lo_name, file, line));
KASSERT(!rm_destroyed(rm),
("rm_rlock() of destroyed rmlock @ %s:%d", file, line));
if (!trylock) {
KASSERT(!rm_wowned(rm),
("rm_rlock: wlock already held for %s @ %s:%d",
rm->lock_object.lo_name, file, line));
WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line,
NULL);
}
if (_rm_rlock(rm, tracker, trylock)) {
if (trylock)
LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 1, file,
line);
else
LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file,
line);
WITNESS_LOCK(&rm->lock_object, 0, file, line);
TD_LOCKS_INC(curthread);
return (1);
} else if (trylock)
LOCK_LOG_TRY("RMRLOCK", &rm->lock_object, 0, 0, file, line);
return (0);
}
2009-04-26 21:16:03 +00:00
void
_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
2009-04-26 21:16:03 +00:00
const char *file, int line)
{
panic: add a switch and infrastructure for stopping other CPUs in SMP case Historical behavior of letting other CPUs merily go on is a default for time being. The new behavior can be switched on via kern.stop_scheduler_on_panic tunable and sysctl. Stopping of the CPUs has (at least) the following benefits: - more of the system state at panic time is preserved intact - threads and interrupts do not interfere with dumping of the system state Only one thread runs uninterrupted after panic if stop_scheduler_on_panic is set. That thread might call code that is also used in normal context and that code might use locks to prevent concurrent execution of certain parts. Those locks might be held by the stopped threads and would never be released. To work around this issue, it was decided that instead of explicit checks for panic context, we would rather put those checks inside the locking primitives. This change has substantial portions written and re-written by attilio and kib at various times. Other changes are heavily based on the ideas and patches submitted by jhb and mdf. bde has provided many insights into the details and history of the current code. The new behavior may cause problems for systems that use a USB keyboard for interfacing with system console. This is because of some unusual locking patterns in the ukbd code which have to be used because on one hand ukbd is below syscons, but on the other hand it has to interface with other usb code that uses regular mutexes/Giant for its concurrency protection. Dumping to USB-connected disks may also be affected. PR: amd64/139614 (at least) In cooperation with: attilio, jhb, kib, mdf Discussed with: arch@, bde Tested by: Eugene Grosbein <eugen@grosbein.net>, gnn, Steven Hartland <killing@multiplay.co.uk>, glebius, Andrew Boyer <aboyer@averesystems.com> (various versions of the patch) MFC after: 3 months (or never)
2011-12-11 21:02:01 +00:00
if (SCHEDULER_STOPPED())
return;
KASSERT(!rm_destroyed(rm),
("rm_runlock() of destroyed rmlock @ %s:%d", file, line));
_rm_assert(rm, RA_RLOCKED, file, line);
2009-04-26 21:16:03 +00:00
WITNESS_UNLOCK(&rm->lock_object, 0, file, line);
LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line);
_rm_runlock(rm, tracker);
TD_LOCKS_DEC(curthread);
}
#else
2009-04-26 21:16:03 +00:00
/*
* Just strip out file and line arguments if no lock debugging is enabled in
* the kernel - we are called from a kernel module.
*/
void
_rm_wlock_debug(struct rmlock *rm, const char *file, int line)
{
2009-04-26 21:16:03 +00:00
_rm_wlock(rm);
}
2009-04-26 21:16:03 +00:00
void
_rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
{
2009-04-26 21:16:03 +00:00
_rm_wunlock(rm);
2009-04-26 21:16:03 +00:00
}
int
_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
int trylock, const char *file, int line)
{
2009-04-26 21:16:03 +00:00
return _rm_rlock(rm, tracker, trylock);
}
2009-04-26 21:16:03 +00:00
void
_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
2009-05-29 14:25:51 +00:00
const char *file, int line)
{
2009-04-26 21:16:03 +00:00
_rm_runlock(rm, tracker);
}
#endif
#ifdef INVARIANT_SUPPORT
#ifndef INVARIANTS
#undef _rm_assert
#endif
/*
* Note that this does not need to use witness_assert() for read lock
* assertions since an exact count of read locks held by this thread
* is computable.
*/
void
_rm_assert(const struct rmlock *rm, int what, const char *file, int line)
{
int count;
if (panicstr != NULL)
return;
switch (what) {
case RA_LOCKED:
case RA_LOCKED | RA_RECURSED:
case RA_LOCKED | RA_NOTRECURSED:
case RA_RLOCKED:
case RA_RLOCKED | RA_RECURSED:
case RA_RLOCKED | RA_NOTRECURSED:
/*
* Handle the write-locked case. Unlike other
* primitives, writers can never recurse.
*/
if (rm_wowned(rm)) {
if (what & RA_RLOCKED)
panic("Lock %s exclusively locked @ %s:%d\n",
rm->lock_object.lo_name, file, line);
if (what & RA_RECURSED)
panic("Lock %s not recursed @ %s:%d\n",
rm->lock_object.lo_name, file, line);
break;
}
critical_enter();
count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
critical_exit();
if (count == 0)
panic("Lock %s not %slocked @ %s:%d\n",
rm->lock_object.lo_name, (what & RA_RLOCKED) ?
"read " : "", file, line);
if (count > 1) {
if (what & RA_NOTRECURSED)
panic("Lock %s recursed @ %s:%d\n",
rm->lock_object.lo_name, file, line);
} else if (what & RA_RECURSED)
panic("Lock %s not recursed @ %s:%d\n",
rm->lock_object.lo_name, file, line);
break;
case RA_WLOCKED:
if (!rm_wowned(rm))
panic("Lock %s not exclusively locked @ %s:%d\n",
rm->lock_object.lo_name, file, line);
break;
case RA_UNLOCKED:
if (rm_wowned(rm))
panic("Lock %s exclusively locked @ %s:%d\n",
rm->lock_object.lo_name, file, line);
critical_enter();
count = rm_trackers_present(pcpu_find(curcpu), rm, curthread);
critical_exit();
if (count != 0)
panic("Lock %s read locked @ %s:%d\n",
rm->lock_object.lo_name, file, line);
break;
default:
panic("Unknown rm lock assertion: %d @ %s:%d", what, file,
line);
}
}
#endif /* INVARIANT_SUPPORT */
#ifdef DDB
static void
print_tracker(struct rm_priotracker *tr)
{
struct thread *td;
td = tr->rmp_thread;
db_printf(" thread %p (tid %d, pid %d, \"%s\") {", td, td->td_tid,
td->td_proc->p_pid, td->td_name);
if (tr->rmp_flags & RMPF_ONQUEUE) {
db_printf("ONQUEUE");
if (tr->rmp_flags & RMPF_SIGNAL)
db_printf(",SIGNAL");
} else
db_printf("0");
db_printf("}\n");
}
static void
db_show_rm(const struct lock_object *lock)
{
struct rm_priotracker *tr;
struct rm_queue *queue;
const struct rmlock *rm;
struct lock_class *lc;
struct pcpu *pc;
rm = (const struct rmlock *)lock;
db_printf(" writecpus: ");
ddb_display_cpuset(__DEQUALIFY(const cpuset_t *, &rm->rm_writecpus));
db_printf("\n");
db_printf(" per-CPU readers:\n");
STAILQ_FOREACH(pc, &cpuhead, pc_allcpu)
for (queue = pc->pc_rm_queue.rmq_next;
queue != &pc->pc_rm_queue; queue = queue->rmq_next) {
tr = (struct rm_priotracker *)queue;
if (tr->rmp_rmlock == rm)
print_tracker(tr);
}
db_printf(" active readers:\n");
LIST_FOREACH(tr, &rm->rm_activeReaders, rmp_qentry)
print_tracker(tr);
lc = LOCK_CLASS(&rm->rm_wlock_object);
db_printf("Backing write-lock (%s):\n", lc->lc_name);
lc->lc_ddb_show(&rm->rm_wlock_object);
}
#endif