2007-11-08 14:47:55 +00:00
|
|
|
/*-
|
|
|
|
* Copyright (c) 2007 Stephan Uphoff <ups@FreeBSD.org>
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
* 3. Neither the name of the author nor the names of any co-contributors
|
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
* without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Machine independent bits of reader/writer lock implementation.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <sys/cdefs.h>
|
|
|
|
__FBSDID("$FreeBSD$");
|
|
|
|
|
|
|
|
#include "opt_ddb.h"
|
2009-05-26 20:28:22 +00:00
|
|
|
#include "opt_kdtrace.h"
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
#include <sys/param.h>
|
|
|
|
#include <sys/systm.h>
|
|
|
|
|
|
|
|
#include <sys/kernel.h>
|
|
|
|
#include <sys/ktr.h>
|
|
|
|
#include <sys/lock.h>
|
|
|
|
#include <sys/mutex.h>
|
|
|
|
#include <sys/proc.h>
|
|
|
|
#include <sys/rmlock.h>
|
|
|
|
#include <sys/sched.h>
|
|
|
|
#include <sys/smp.h>
|
|
|
|
#include <sys/turnstile.h>
|
|
|
|
#include <sys/lock_profile.h>
|
|
|
|
#include <machine/cpu.h>
|
|
|
|
|
|
|
|
#ifdef DDB
|
|
|
|
#include <ddb/ddb.h>
|
|
|
|
#endif
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
#define RMPF_ONQUEUE 1
|
|
|
|
#define RMPF_SIGNAL 2
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/*
|
|
|
|
* To support usage of rmlock in CVs and msleep yet another list for the
|
|
|
|
* priority tracker would be needed. Using this lock for cv and msleep also
|
|
|
|
* does not seem very useful
|
2007-11-08 14:47:55 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
static __inline void compiler_memory_barrier(void) {
|
|
|
|
__asm __volatile("":::"memory");
|
|
|
|
}
|
|
|
|
|
2011-11-16 21:51:17 +00:00
|
|
|
static void assert_rm(const struct lock_object *lock, int what);
|
2009-04-26 21:16:03 +00:00
|
|
|
static void lock_rm(struct lock_object *lock, int how);
|
2009-05-26 20:28:22 +00:00
|
|
|
#ifdef KDTRACE_HOOKS
|
2011-11-16 21:51:17 +00:00
|
|
|
static int owner_rm(const struct lock_object *lock, struct thread **owner);
|
2009-05-26 20:28:22 +00:00
|
|
|
#endif
|
2009-04-26 21:16:03 +00:00
|
|
|
static int unlock_rm(struct lock_object *lock);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
struct lock_class lock_class_rm = {
|
|
|
|
.lc_name = "rm",
|
|
|
|
.lc_flags = LC_SLEEPLOCK | LC_RECURSABLE,
|
2007-11-18 14:43:53 +00:00
|
|
|
.lc_assert = assert_rm,
|
2007-11-08 14:47:55 +00:00
|
|
|
#if 0
|
|
|
|
#ifdef DDB
|
|
|
|
.lc_ddb_show = db_show_rwlock,
|
|
|
|
#endif
|
|
|
|
#endif
|
|
|
|
.lc_lock = lock_rm,
|
|
|
|
.lc_unlock = unlock_rm,
|
2009-05-26 20:28:22 +00:00
|
|
|
#ifdef KDTRACE_HOOKS
|
|
|
|
.lc_owner = owner_rm,
|
|
|
|
#endif
|
2007-11-08 14:47:55 +00:00
|
|
|
};
|
|
|
|
|
2007-11-18 14:43:53 +00:00
|
|
|
static void
|
2011-11-16 21:51:17 +00:00
|
|
|
assert_rm(const struct lock_object *lock, int what)
|
2007-11-18 14:43:53 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
panic("assert_rm called");
|
|
|
|
}
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
static void
|
2009-04-26 21:16:03 +00:00
|
|
|
lock_rm(struct lock_object *lock, int how)
|
|
|
|
{
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
panic("lock_rm called");
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
2009-04-26 21:16:03 +00:00
|
|
|
unlock_rm(struct lock_object *lock)
|
|
|
|
{
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
panic("unlock_rm called");
|
|
|
|
}
|
|
|
|
|
2009-05-26 20:28:22 +00:00
|
|
|
#ifdef KDTRACE_HOOKS
|
|
|
|
static int
|
2011-11-16 21:51:17 +00:00
|
|
|
owner_rm(const struct lock_object *lock, struct thread **owner)
|
2009-05-26 20:28:22 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
panic("owner_rm called");
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
static struct mtx rm_spinlock;
|
|
|
|
|
|
|
|
MTX_SYSINIT(rm_spinlock, &rm_spinlock, "rm_spinlock", MTX_SPIN);
|
|
|
|
|
|
|
|
/*
|
2009-12-25 01:16:24 +00:00
|
|
|
* Add or remove tracker from per-cpu list.
|
2009-04-26 21:16:03 +00:00
|
|
|
*
|
2009-12-25 01:16:24 +00:00
|
|
|
* The per-cpu list can be traversed at any time in forward direction from an
|
2009-04-26 21:16:03 +00:00
|
|
|
* interrupt on the *local* cpu.
|
2007-11-08 14:47:55 +00:00
|
|
|
*/
|
2009-04-26 21:16:03 +00:00
|
|
|
static void inline
|
|
|
|
rm_tracker_add(struct pcpu *pc, struct rm_priotracker *tracker)
|
|
|
|
{
|
|
|
|
struct rm_queue *next;
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
/* Initialize all tracker pointers */
|
|
|
|
tracker->rmp_cpuQueue.rmq_prev = &pc->pc_rm_queue;
|
|
|
|
next = pc->pc_rm_queue.rmq_next;
|
|
|
|
tracker->rmp_cpuQueue.rmq_next = next;
|
2009-04-26 21:16:03 +00:00
|
|
|
|
|
|
|
/* rmq_prev is not used during froward traversal. */
|
2007-11-08 14:47:55 +00:00
|
|
|
next->rmq_prev = &tracker->rmp_cpuQueue;
|
2009-04-26 21:16:03 +00:00
|
|
|
|
|
|
|
/* Update pointer to first element. */
|
2009-12-25 21:14:05 +00:00
|
|
|
pc->pc_rm_queue.rmq_next = &tracker->rmp_cpuQueue;
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
static void inline
|
|
|
|
rm_tracker_remove(struct pcpu *pc, struct rm_priotracker *tracker)
|
|
|
|
{
|
2007-11-08 14:47:55 +00:00
|
|
|
struct rm_queue *next, *prev;
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
next = tracker->rmp_cpuQueue.rmq_next;
|
|
|
|
prev = tracker->rmp_cpuQueue.rmq_prev;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/* Not used during forward traversal. */
|
|
|
|
next->rmq_prev = prev;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/* Remove from list. */
|
|
|
|
prev->rmq_next = next;
|
|
|
|
}
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
static void
|
|
|
|
rm_cleanIPI(void *arg)
|
|
|
|
{
|
2007-11-08 14:47:55 +00:00
|
|
|
struct pcpu *pc;
|
2009-04-26 21:16:03 +00:00
|
|
|
struct rmlock *rm = arg;
|
2007-11-08 14:47:55 +00:00
|
|
|
struct rm_priotracker *tracker;
|
2009-04-26 21:16:03 +00:00
|
|
|
struct rm_queue *queue;
|
2007-11-08 14:47:55 +00:00
|
|
|
pc = pcpu_find(curcpu);
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
for (queue = pc->pc_rm_queue.rmq_next; queue != &pc->pc_rm_queue;
|
2007-11-08 14:47:55 +00:00
|
|
|
queue = queue->rmq_next) {
|
2009-04-26 21:16:03 +00:00
|
|
|
tracker = (struct rm_priotracker *)queue;
|
|
|
|
if (tracker->rmp_rmlock == rm && tracker->rmp_flags == 0) {
|
2007-11-08 14:47:55 +00:00
|
|
|
tracker->rmp_flags = RMPF_ONQUEUE;
|
|
|
|
mtx_lock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
LIST_INSERT_HEAD(&rm->rm_activeReaders, tracker,
|
2007-11-08 14:47:55 +00:00
|
|
|
rmp_qentry);
|
|
|
|
mtx_unlock_spin(&rm_spinlock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
CTASSERT((RM_SLEEPABLE & LO_CLASSFLAGS) == RM_SLEEPABLE);
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
void
|
2009-05-29 10:52:37 +00:00
|
|
|
rm_init_flags(struct rmlock *rm, const char *name, int opts)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
2009-05-29 10:52:37 +00:00
|
|
|
int liflags;
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2009-05-29 10:52:37 +00:00
|
|
|
liflags = 0;
|
|
|
|
if (!(opts & RM_NOWITNESS))
|
|
|
|
liflags |= LO_WITNESS;
|
|
|
|
if (opts & RM_RECURSE)
|
|
|
|
liflags |= LO_RECURSABLE;
|
2010-09-01 19:50:03 +00:00
|
|
|
rm->rm_writecpus = all_cpus;
|
2007-11-08 14:47:55 +00:00
|
|
|
LIST_INIT(&rm->rm_activeReaders);
|
2010-09-01 19:50:03 +00:00
|
|
|
if (opts & RM_SLEEPABLE) {
|
|
|
|
liflags |= RM_SLEEPABLE;
|
|
|
|
sx_init_flags(&rm->rm_lock_sx, "rmlock_sx", SX_RECURSE);
|
|
|
|
} else
|
|
|
|
mtx_init(&rm->rm_lock_mtx, name, "rmlock_mtx", MTX_NOWITNESS);
|
2009-05-29 10:52:37 +00:00
|
|
|
lock_init(&rm->lock_object, &lock_class_rm, name, NULL, liflags);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
rm_init(struct rmlock *rm, const char *name)
|
|
|
|
{
|
|
|
|
|
|
|
|
rm_init_flags(rm, name, 0);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
rm_destroy(struct rmlock *rm)
|
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
sx_destroy(&rm->rm_lock_sx);
|
|
|
|
else
|
|
|
|
mtx_destroy(&rm->rm_lock_mtx);
|
2007-11-08 14:47:55 +00:00
|
|
|
lock_destroy(&rm->lock_object);
|
|
|
|
}
|
|
|
|
|
2007-11-10 15:06:30 +00:00
|
|
|
int
|
2011-11-16 21:51:17 +00:00
|
|
|
rm_wowned(const struct rmlock *rm)
|
2007-11-10 15:06:30 +00:00
|
|
|
{
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
return (sx_xlocked(&rm->rm_lock_sx));
|
|
|
|
else
|
|
|
|
return (mtx_owned(&rm->rm_lock_mtx));
|
2007-11-10 15:06:30 +00:00
|
|
|
}
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
void
|
|
|
|
rm_sysinit(void *arg)
|
|
|
|
{
|
|
|
|
struct rm_args *args = arg;
|
2009-05-29 10:52:37 +00:00
|
|
|
|
|
|
|
rm_init(args->ra_rm, args->ra_desc);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
rm_sysinit_flags(void *arg)
|
|
|
|
{
|
|
|
|
struct rm_args_flags *args = arg;
|
|
|
|
|
|
|
|
rm_init_flags(args->ra_rm, args->ra_desc, args->ra_opts);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
static int
|
|
|
|
_rm_rlock_hard(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
|
|
|
struct pcpu *pc;
|
|
|
|
struct rm_queue *queue;
|
2009-04-26 21:16:03 +00:00
|
|
|
struct rm_priotracker *atracker;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
critical_enter();
|
|
|
|
pc = pcpu_find(curcpu);
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/* Check if we just need to do a proper critical_exit. */
|
2011-06-13 13:28:31 +00:00
|
|
|
if (!CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)) {
|
2007-11-08 14:47:55 +00:00
|
|
|
critical_exit();
|
2010-09-01 19:50:03 +00:00
|
|
|
return (1);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
2009-12-25 01:16:24 +00:00
|
|
|
/* Remove our tracker from the per-cpu list. */
|
2009-04-26 21:16:03 +00:00
|
|
|
rm_tracker_remove(pc, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/* Check to see if the IPI granted us the lock after all. */
|
|
|
|
if (tracker->rmp_flags) {
|
|
|
|
/* Just add back tracker - we hold the lock. */
|
|
|
|
rm_tracker_add(pc, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
critical_exit();
|
2010-09-01 19:50:03 +00:00
|
|
|
return (1);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2009-04-26 21:16:03 +00:00
|
|
|
* We allow readers to aquire a lock even if a writer is blocked if
|
|
|
|
* the lock is recursive and the reader already holds the lock.
|
2007-11-08 14:47:55 +00:00
|
|
|
*/
|
|
|
|
if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) {
|
|
|
|
/*
|
2009-12-25 01:16:24 +00:00
|
|
|
* Just grant the lock if this thread already has a tracker
|
|
|
|
* for this lock on the per-cpu queue.
|
2007-11-08 14:47:55 +00:00
|
|
|
*/
|
2009-04-26 21:16:03 +00:00
|
|
|
for (queue = pc->pc_rm_queue.rmq_next;
|
2009-12-25 21:14:05 +00:00
|
|
|
queue != &pc->pc_rm_queue; queue = queue->rmq_next) {
|
2009-04-26 21:16:03 +00:00
|
|
|
atracker = (struct rm_priotracker *)queue;
|
|
|
|
if ((atracker->rmp_rmlock == rm) &&
|
|
|
|
(atracker->rmp_thread == tracker->rmp_thread)) {
|
2007-11-08 14:47:55 +00:00
|
|
|
mtx_lock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
LIST_INSERT_HEAD(&rm->rm_activeReaders,
|
|
|
|
tracker, rmp_qentry);
|
2007-11-08 14:47:55 +00:00
|
|
|
tracker->rmp_flags = RMPF_ONQUEUE;
|
|
|
|
mtx_unlock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
rm_tracker_add(pc, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
critical_exit();
|
2010-09-01 19:50:03 +00:00
|
|
|
return (1);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
sched_unpin();
|
|
|
|
critical_exit();
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (trylock) {
|
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE) {
|
|
|
|
if (!sx_try_xlock(&rm->rm_lock_sx))
|
|
|
|
return (0);
|
|
|
|
} else {
|
|
|
|
if (!mtx_trylock(&rm->rm_lock_mtx))
|
|
|
|
return (0);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
sx_xlock(&rm->rm_lock_sx);
|
|
|
|
else
|
|
|
|
mtx_lock(&rm->rm_lock_mtx);
|
|
|
|
}
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
critical_enter();
|
2007-11-08 14:47:55 +00:00
|
|
|
pc = pcpu_find(curcpu);
|
2011-06-13 13:28:31 +00:00
|
|
|
CPU_CLR(pc->pc_cpuid, &rm->rm_writecpus);
|
2009-04-26 21:16:03 +00:00
|
|
|
rm_tracker_add(pc, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
sched_pin();
|
|
|
|
critical_exit();
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
sx_xunlock(&rm->rm_lock_sx);
|
|
|
|
else
|
|
|
|
mtx_unlock(&rm->rm_lock_mtx);
|
|
|
|
|
|
|
|
return (1);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
int
|
|
|
|
_rm_rlock(struct rmlock *rm, struct rm_priotracker *tracker, int trylock)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
|
|
|
struct thread *td = curthread;
|
|
|
|
struct pcpu *pc;
|
|
|
|
|
|
|
|
tracker->rmp_flags = 0;
|
|
|
|
tracker->rmp_thread = td;
|
|
|
|
tracker->rmp_rmlock = rm;
|
|
|
|
|
|
|
|
td->td_critnest++; /* critical_enter(); */
|
|
|
|
|
|
|
|
compiler_memory_barrier();
|
|
|
|
|
|
|
|
pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
rm_tracker_add(pc, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-05-29 14:20:10 +00:00
|
|
|
sched_pin();
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
compiler_memory_barrier();
|
|
|
|
|
|
|
|
td->td_critnest--;
|
2009-04-26 21:16:03 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Fast path to combine two common conditions into a single
|
|
|
|
* conditional jump.
|
2007-11-08 14:47:55 +00:00
|
|
|
*/
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
if (0 == (td->td_owepreempt |
|
2011-06-13 13:28:31 +00:00
|
|
|
CPU_ISSET(pc->pc_cpuid, &rm->rm_writecpus)))
|
2010-09-01 19:50:03 +00:00
|
|
|
return (1);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/* We do not have a read token and need to acquire one. */
|
2010-09-01 19:50:03 +00:00
|
|
|
return _rm_rlock_hard(rm, tracker, trylock);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2009-04-26 21:16:03 +00:00
|
|
|
_rm_unlock_hard(struct thread *td,struct rm_priotracker *tracker)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
if (td->td_owepreempt) {
|
|
|
|
td->td_critnest++;
|
|
|
|
critical_exit();
|
|
|
|
}
|
2009-04-26 21:16:03 +00:00
|
|
|
|
|
|
|
if (!tracker->rmp_flags)
|
2007-11-08 14:47:55 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
mtx_lock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
LIST_REMOVE(tracker, rmp_qentry);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
if (tracker->rmp_flags & RMPF_SIGNAL) {
|
|
|
|
struct rmlock *rm;
|
2009-04-26 21:16:03 +00:00
|
|
|
struct turnstile *ts;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
rm = tracker->rmp_rmlock;
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
turnstile_chain_lock(&rm->lock_object);
|
|
|
|
mtx_unlock_spin(&rm_spinlock);
|
|
|
|
|
|
|
|
ts = turnstile_lookup(&rm->lock_object);
|
|
|
|
|
|
|
|
turnstile_signal(ts, TS_EXCLUSIVE_QUEUE);
|
|
|
|
turnstile_unpend(ts, TS_EXCLUSIVE_LOCK);
|
|
|
|
turnstile_chain_unlock(&rm->lock_object);
|
|
|
|
} else
|
|
|
|
mtx_unlock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
}
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
void
|
2009-04-26 21:16:03 +00:00
|
|
|
_rm_runlock(struct rmlock *rm, struct rm_priotracker *tracker)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
|
|
|
struct pcpu *pc;
|
|
|
|
struct thread *td = tracker->rmp_thread;
|
|
|
|
|
|
|
|
td->td_critnest++; /* critical_enter(); */
|
|
|
|
pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */
|
2009-04-26 21:16:03 +00:00
|
|
|
rm_tracker_remove(pc, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
td->td_critnest--;
|
2009-05-29 14:20:10 +00:00
|
|
|
sched_unpin();
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
if (0 == (td->td_owepreempt | tracker->rmp_flags))
|
2007-11-08 14:47:55 +00:00
|
|
|
return;
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
_rm_unlock_hard(td, tracker);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
_rm_wlock(struct rmlock *rm)
|
|
|
|
{
|
|
|
|
struct rm_priotracker *prio;
|
|
|
|
struct turnstile *ts;
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
cpuset_t readcpus;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
sx_xlock(&rm->rm_lock_sx);
|
|
|
|
else
|
|
|
|
mtx_lock(&rm->rm_lock_mtx);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
if (CPU_CMP(&rm->rm_writecpus, &all_cpus)) {
|
2007-11-08 14:47:55 +00:00
|
|
|
/* Get all read tokens back */
|
Commit the support for removing cpumask_t and replacing it directly with
cpuset_t objects.
That is going to offer the underlying support for a simple bump of
MAXCPU and then support for number of cpus > 32 (as it is today).
Right now, cpumask_t is an int, 32 bits on all our supported architecture.
cpumask_t on the other side is implemented as an array of longs, and
easilly extendible by definition.
The architectures touched by this commit are the following:
- amd64
- i386
- pc98
- arm
- ia64
- XEN
while the others are still missing.
Userland is believed to be fully converted with the changes contained
here.
Some technical notes:
- This commit may be considered an ABI nop for all the architectures
different from amd64 and ia64 (and sparc64 in the future)
- per-cpu members, which are now converted to cpuset_t, needs to be
accessed avoiding migration, because the size of cpuset_t should be
considered unknown
- size of cpuset_t objects is different from kernel and userland (this is
primirally done in order to leave some more space in userland to cope
with KBI extensions). If you need to access kernel cpuset_t from the
userland please refer to example in this patch on how to do that
correctly (kgdb may be a good source, for example).
- Support for other architectures is going to be added soon
- Only MAXCPU for amd64 is bumped now
The patch has been tested by sbruno and Nicholas Esborn on opteron
4 x 12 pack CPUs. More testing on big SMP is expected to came soon.
pluknet tested the patch with his 8-ways on both amd64 and i386.
Tested by: pluknet, sbruno, gianni, Nicholas Esborn
Reviewed by: jeff, jhb, sbruno
2011-05-05 14:39:14 +00:00
|
|
|
readcpus = all_cpus;
|
|
|
|
CPU_NAND(&readcpus, &rm->rm_writecpus);
|
2010-09-01 19:50:03 +00:00
|
|
|
rm->rm_writecpus = all_cpus;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/*
|
2010-09-01 19:50:03 +00:00
|
|
|
* Assumes rm->rm_writecpus update is visible on other CPUs
|
2009-04-26 21:16:03 +00:00
|
|
|
* before rm_cleanIPI is called.
|
2007-11-08 14:47:55 +00:00
|
|
|
*/
|
|
|
|
#ifdef SMP
|
2010-09-01 19:50:03 +00:00
|
|
|
smp_rendezvous_cpus(readcpus,
|
|
|
|
smp_no_rendevous_barrier,
|
2007-11-08 14:47:55 +00:00
|
|
|
rm_cleanIPI,
|
2009-04-26 21:16:03 +00:00
|
|
|
smp_no_rendevous_barrier,
|
|
|
|
rm);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
#else
|
|
|
|
rm_cleanIPI(rm);
|
|
|
|
#endif
|
|
|
|
|
|
|
|
mtx_lock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
while ((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) {
|
2007-11-08 14:47:55 +00:00
|
|
|
ts = turnstile_trywait(&rm->lock_object);
|
|
|
|
prio->rmp_flags = RMPF_ONQUEUE | RMPF_SIGNAL;
|
|
|
|
mtx_unlock_spin(&rm_spinlock);
|
2009-04-26 21:16:03 +00:00
|
|
|
turnstile_wait(ts, prio->rmp_thread,
|
|
|
|
TS_EXCLUSIVE_QUEUE);
|
2007-11-08 14:47:55 +00:00
|
|
|
mtx_lock_spin(&rm_spinlock);
|
|
|
|
}
|
|
|
|
mtx_unlock_spin(&rm_spinlock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
_rm_wunlock(struct rmlock *rm)
|
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
sx_xunlock(&rm->rm_lock_sx);
|
|
|
|
else
|
|
|
|
mtx_unlock(&rm->rm_lock_mtx);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef LOCK_DEBUG
|
|
|
|
|
|
|
|
void _rm_wlock_debug(struct rmlock *rm, const char *file, int line)
|
|
|
|
{
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE,
|
2008-09-10 19:13:30 +00:00
|
|
|
file, line, NULL);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
_rm_wlock(rm);
|
|
|
|
|
|
|
|
LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line);
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
WITNESS_LOCK(&rm->rm_lock_sx.lock_object, LOP_EXCLUSIVE,
|
|
|
|
file, line);
|
|
|
|
else
|
|
|
|
WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
|
|
|
curthread->td_locks++;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
void
|
|
|
|
_rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
curthread->td_locks--;
|
2010-09-01 19:50:03 +00:00
|
|
|
if (rm->lock_object.lo_flags & RM_SLEEPABLE)
|
|
|
|
WITNESS_UNLOCK(&rm->rm_lock_sx.lock_object, LOP_EXCLUSIVE,
|
|
|
|
file, line);
|
|
|
|
else
|
|
|
|
WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line);
|
2007-11-08 14:47:55 +00:00
|
|
|
LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line);
|
|
|
|
_rm_wunlock(rm);
|
2009-04-26 21:16:03 +00:00
|
|
|
}
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
int
|
2007-11-08 14:47:55 +00:00
|
|
|
_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
|
2010-09-01 19:50:03 +00:00
|
|
|
int trylock, const char *file, int line)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
2010-09-01 19:50:03 +00:00
|
|
|
if (!trylock && (rm->lock_object.lo_flags & RM_SLEEPABLE))
|
|
|
|
WITNESS_CHECKORDER(&rm->rm_lock_sx.lock_object, LOP_NEWORDER,
|
|
|
|
file, line, NULL);
|
2009-04-26 21:16:03 +00:00
|
|
|
WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line, NULL);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
if (_rm_rlock(rm, tracker, trylock)) {
|
|
|
|
LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file, line);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
WITNESS_LOCK(&rm->lock_object, 0, file, line);
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
curthread->td_locks++;
|
2007-11-08 14:47:55 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
return (1);
|
|
|
|
}
|
|
|
|
|
|
|
|
return (0);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
void
|
2009-12-25 21:14:05 +00:00
|
|
|
_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
|
2009-04-26 21:16:03 +00:00
|
|
|
const char *file, int line)
|
|
|
|
{
|
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
curthread->td_locks--;
|
2009-04-26 21:16:03 +00:00
|
|
|
WITNESS_UNLOCK(&rm->lock_object, 0, file, line);
|
2007-11-08 14:47:55 +00:00
|
|
|
LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line);
|
|
|
|
_rm_runlock(rm, tracker);
|
|
|
|
}
|
|
|
|
|
|
|
|
#else
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
/*
|
|
|
|
* Just strip out file and line arguments if no lock debugging is enabled in
|
|
|
|
* the kernel - we are called from a kernel module.
|
|
|
|
*/
|
|
|
|
void
|
|
|
|
_rm_wlock_debug(struct rmlock *rm, const char *file, int line)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
_rm_wlock(rm);
|
|
|
|
}
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
void
|
|
|
|
_rm_wunlock_debug(struct rmlock *rm, const char *file, int line)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
_rm_wunlock(rm);
|
2009-04-26 21:16:03 +00:00
|
|
|
}
|
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
int
|
2007-11-08 14:47:55 +00:00
|
|
|
_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
|
2010-09-01 19:50:03 +00:00
|
|
|
int trylock, const char *file, int line)
|
2007-11-08 14:47:55 +00:00
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2010-09-01 19:50:03 +00:00
|
|
|
return _rm_rlock(rm, tracker, trylock);
|
2007-11-08 14:47:55 +00:00
|
|
|
}
|
|
|
|
|
2009-04-26 21:16:03 +00:00
|
|
|
void
|
2009-12-25 21:14:05 +00:00
|
|
|
_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker,
|
2009-05-29 14:25:51 +00:00
|
|
|
const char *file, int line)
|
|
|
|
{
|
2009-04-26 21:16:03 +00:00
|
|
|
|
2007-11-08 14:47:55 +00:00
|
|
|
_rm_runlock(rm, tracker);
|
|
|
|
}
|
|
|
|
|
|
|
|
#endif
|