d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Trim another pointer from struct lock_object (and thus from struct mtx and

Browse Source 
struct sx).  Instead of storing a direct pointer to a our lock_class
struct in lock_object, reserve 4 bits in the lo_flags field to serve as an
index into a global lock_classes array that contains pointers to the lock
classes.  Only debugging code such as WITNESS or INVARIANTS checks and KTR
logging need to access the lock_class member, so this shouldn't add any
overhead to production kernels.  It might add some slight overhead to
kernels using those debug options however.

As with the previous set of changes to lock_object, this is going to
completely obliterate the kernel ABI, so be sure to recompile all your
modules.
This commit is contained in:
John Baldwin 2006-01-06 18:07:32 +00:00
parent ca49f12fdb
commit 3c6decc327
5 changed files with 82 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
sys/kern/kern_mutex.c
View File

@ -278,7 +278,7 @@ _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line)
{
MPASS(curthread != NULL);
KASSERT(m->mtx_object.lo_class == &lock_class_mtx_sleep,
KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SLEEP_MUTEX,
("mtx_lock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name,
file, line));
WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
@ -303,7 +303,7 @@ _mtx_unlock_flags(struct mtx *m, int opts, const char *file, int line)
{
MPASS(curthread != NULL);
KASSERT(m->mtx_object.lo_class == &lock_class_mtx_sleep,
KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SLEEP_MUTEX,
("mtx_unlock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name,
file, line));
WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line);
@ -382,7 +382,7 @@ _mtx_lock_spin_flags(struct mtx *m, int opts, const char *file, int line)
{
MPASS(curthread != NULL);
KASSERT(m->mtx_object.lo_class == &lock_class_mtx_spin,
KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SPIN_MUTEX,
("mtx_lock_spin() of sleep mutex %s @ %s:%d",
m->mtx_object.lo_name, file, line));
WITNESS_CHECKORDER(&m->mtx_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE,
@ -398,7 +398,7 @@ _mtx_unlock_spin_flags(struct mtx *m, int opts, const char *file, int line)
{
MPASS(curthread != NULL);
KASSERT(m->mtx_object.lo_class == &lock_class_mtx_spin,
KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SPIN_MUTEX,
("mtx_unlock_spin() of sleep mutex %s @ %s:%d",
m->mtx_object.lo_name, file, line));
WITNESS_UNLOCK(&m->mtx_object, opts | LOP_EXCLUSIVE, file, line);
@ -419,7 +419,7 @@ _mtx_trylock(struct mtx *m, int opts, const char *file, int line)
int rval;
MPASS(curthread != NULL);
KASSERT(m->mtx_object.lo_class == &lock_class_mtx_sleep,
KASSERT(LO_CLASSINDEX(&m->mtx_object) == LOCK_CLASS_SLEEP_MUTEX,
("mtx_trylock() of spin mutex %s @ %s:%d", m->mtx_object.lo_name,
file, line));
@ -845,13 +845,13 @@ mtx_init(struct mtx *m, const char *name, const char *type, int opts)
("mutex \"%s\" %p already initialized", name, m));
bzero(m, sizeof(*m));
if (opts & MTX_SPIN)
lock->lo_class = &lock_class_mtx_spin;
lock->lo_flags = LOCK_CLASS_SPIN_MUTEX << LO_CLASSSHIFT;
else
lock->lo_class = &lock_class_mtx_sleep;
lock->lo_flags = LOCK_CLASS_SLEEP_MUTEX << LO_CLASSSHIFT;
lock->lo_name = name;
lock->lo_type = type != NULL ? type : name;
if (opts & MTX_QUIET)
lock->lo_flags = LO_QUIET;
lock->lo_flags |= LO_QUIET;
if (opts & MTX_RECURSE)
lock->lo_flags |= LO_RECURSABLE;
if ((opts & MTX_NOWITNESS) == 0)
@ -883,6 +883,10 @@ mtx_destroy(struct mtx *m)
else {
MPASS((m->mtx_lock & (MTX_RECURSED|MTX_CONTESTED)) == 0);
/* Perform the non-mtx related part of mtx_unlock_spin(). */
if (m->mtx_object.lo_class == &lock_class_mtx_spin)
spinlock_exit();
/* Tell witness this isn't locked to make it happy. */
WITNESS_UNLOCK(&m->mtx_object, LOP_EXCLUSIVE, __FILE__,
__LINE__);
@ -916,26 +920,35 @@ mutex_init(void)
mtx_lock(&Giant);
}
#if LOCK_DEBUG > 0 || defined(DDB)
/* XXX: This is not mutex-specific. */
struct lock_class *lock_classes[LOCK_CLASS_MAX + 1] = {
&lock_class_mtx_spin,
&lock_class_mtx_sleep,
&lock_class_sx,
};
#endif
#ifdef DDB
/* XXX: This function is not mutex-specific. */
DB_SHOW_COMMAND(lock, db_show_lock)
{
struct lock_object *lock;
struct lock_class *class;
if (!have_addr)
return;
lock = (struct lock_object *)addr;
if (lock->lo_class != &lock_class_mtx_sleep &&
lock->lo_class != &lock_class_mtx_spin &&
lock->lo_class != &lock_class_sx) {
db_printf("Unknown lock class\n");
if (LO_CLASSINDEX(lock) > LOCK_CLASS_MAX) {
db_printf("Unknown lock class: %d\n", LO_CLASSINDEX(lock));
return;
}
db_printf(" class: %s\n", lock->lo_class->lc_name);
class = LOCK_CLASS(lock);
db_printf(" class: %s\n", class->lc_name);
db_printf(" name: %s\n", lock->lo_name);
if (lock->lo_type && lock->lo_type != lock->lo_name)
db_printf(" type: %s\n", lock->lo_type);
lock->lo_class->lc_ddb_show(lock);
class->lc_ddb_show(lock);
}
void
@ -947,7 +960,7 @@ db_show_mtx(struct lock_object *lock)
m = (struct mtx *)lock;
db_printf(" flags: {");
if (m->mtx_object.lo_class == &lock_class_mtx_spin)
if (LO_CLASSINDEX(lock) == LOCK_CLASS_SPIN_MUTEX)
db_printf("SPIN");
else
db_printf("DEF");

4
sys/kern/kern_sx.c
View File

@ -83,9 +83,9 @@ sx_init(struct sx *sx, const char *description)
KASSERT((lock->lo_flags & LO_INITIALIZED) == 0,
("sx lock %s %p already initialized", description, sx));
bzero(sx, sizeof(*sx));
lock->lo_class = &lock_class_sx;
lock->lo_flags = LOCK_CLASS_SX << LO_CLASSSHIFT;
lock->lo_type = lock->lo_name = description;
lock->lo_flags = LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE |
lock->lo_flags |= LO_WITNESS | LO_RECURSABLE | LO_SLEEPABLE |
LO_UPGRADABLE;
sx->sx_lock = mtx_pool_find(mtxpool_lockbuilder, sx);
sx->sx_cnt = 0;

58
sys/kern/subr_witness.c
View File

@ -506,7 +506,7 @@ witness_initialize(void *dummy __unused)
KASSERT(lock->lo_flags & LO_WITNESS,
("%s: lock %s is on pending list but not LO_WITNESS",
__func__, lock->lo_name));
lock->lo_witness = enroll(lock->lo_type, lock->lo_class);
lock->lo_witness = enroll(lock->lo_type, LOCK_CLASS(lock));
}
/* Mark the witness code as being ready for use. */
@ -542,7 +542,7 @@ witness_init(struct lock_object *lock)
struct lock_class *class;
/* Various sanity checks. */
class = lock->lo_class;
class = LOCK_CLASS(lock);
if (lock->lo_flags & LO_INITIALIZED)
panic("%s: lock (%s) %s is already initialized", __func__,
class->lc_name, lock->lo_name);
@ -580,14 +580,16 @@ witness_init(struct lock_object *lock)
void
witness_destroy(struct lock_object *lock)
{
struct lock_class *class;
struct witness *w;
class = LOCK_CLASS(lock);
if (witness_cold)
panic("lock (%s) %s destroyed while witness_cold",
lock->lo_class->lc_name, lock->lo_name);
class->lc_name, lock->lo_name);
if ((lock->lo_flags & LO_INITIALIZED) == 0)
panic("%s: lock (%s) %s is not initialized", __func__,
lock->lo_class->lc_name, lock->lo_name);
class->lc_name, lock->lo_name);
/* XXX: need to verify that no one holds the lock */
if ((lock->lo_flags & (LO_WITNESS | LO_ENROLLPEND)) == LO_WITNESS &&
@ -824,7 +826,7 @@ witness_checkorder(struct lock_object *lock, int flags, const char *file,
__func__);
w = lock->lo_witness;
class = lock->lo_class;
class = LOCK_CLASS(lock);
td = curthread;
file = fixup_filename(file);
@ -1104,7 +1106,7 @@ witness_lock(struct lock_object *lock, int flags, const char *file, int line)
file = fixup_filename(file);
/* Determine lock list for this lock. */
if (lock->lo_class->lc_flags & LC_SLEEPLOCK)
if (LOCK_CLASS(lock)->lc_flags & LC_SLEEPLOCK)
lock_list = &td->td_sleeplocks;
else
lock_list = PCPU_PTR(spinlocks);
@ -1157,7 +1159,7 @@ witness_upgrade(struct lock_object *lock, int flags, const char *file, int line)
KASSERT(!witness_cold, ("%s: witness_cold", __func__));
if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL)
return;
class = lock->lo_class;
class = LOCK_CLASS(lock);
file = fixup_filename(file);
if ((lock->lo_flags & LO_UPGRADABLE) == 0)
panic("upgrade of non-upgradable lock (%s) %s @ %s:%d",
@ -1165,7 +1167,7 @@ witness_upgrade(struct lock_object *lock, int flags, const char *file, int line)
if ((flags & LOP_TRYLOCK) == 0)
panic("non-try upgrade of lock (%s) %s @ %s:%d", class->lc_name,
lock->lo_name, file, line);
if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0)
if ((class->lc_flags & LC_SLEEPLOCK) == 0)
panic("upgrade of non-sleep lock (%s) %s @ %s:%d",
class->lc_name, lock->lo_name, file, line);
instance = find_instance(curthread->td_sleeplocks, lock);
@ -1192,12 +1194,12 @@ witness_downgrade(struct lock_object *lock, int flags, const char *file,
KASSERT(!witness_cold, ("%s: witness_cold", __func__));
if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL)
return;
class = lock->lo_class;
class = LOCK_CLASS(lock);
file = fixup_filename(file);
if ((lock->lo_flags & LO_UPGRADABLE) == 0)
panic("downgrade of non-upgradable lock (%s) %s @ %s:%d",
class->lc_name, lock->lo_name, file, line);
if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0)
if ((class->lc_flags & LC_SLEEPLOCK) == 0)
panic("downgrade of non-sleep lock (%s) %s @ %s:%d",
class->lc_name, lock->lo_name, file, line);
instance = find_instance(curthread->td_sleeplocks, lock);
@ -1228,7 +1230,7 @@ witness_unlock(struct lock_object *lock, int flags, const char *file, int line)
panicstr != NULL)
return;
td = curthread;
class = lock->lo_class;
class = LOCK_CLASS(lock);
file = fixup_filename(file);
/* Find lock instance associated with this lock. */
@ -1739,7 +1741,7 @@ witness_list_lock(struct lock_instance *instance)
lock = instance->li_lock;
printf("%s %s %s", (instance->li_flags & LI_EXCLUSIVE) != 0 ?
"exclusive" : "shared", lock->lo_class->lc_name, lock->lo_name);
"exclusive" : "shared", LOCK_CLASS(lock)->lc_name, lock->lo_name);
if (lock->lo_type != lock->lo_name)
printf(" (%s)", lock->lo_type);
printf(" r = %d (%p) locked @ %s:%d\n",
@ -1809,11 +1811,13 @@ witness_save(struct lock_object *lock, const char **filep, int *linep)
{
struct lock_list_entry *lock_list;
struct lock_instance *instance;
struct lock_class *class;
KASSERT(!witness_cold, ("%s: witness_cold", __func__));
if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL)
return;
if (lock->lo_class->lc_flags & LC_SLEEPLOCK)
class = LOCK_CLASS(lock);
if (class->lc_flags & LC_SLEEPLOCK)
lock_list = curthread->td_sleeplocks;
else {
if (witness_skipspin)
@ -1823,7 +1827,7 @@ witness_save(struct lock_object *lock, const char **filep, int *linep)
instance = find_instance(lock_list, lock);
if (instance == NULL)
panic("%s: lock (%s) %s not locked", __func__,
lock->lo_class->lc_name, lock->lo_name);
class->lc_name, lock->lo_name);
*filep = instance->li_file;
*linep = instance->li_line;
}
@ -1833,11 +1837,13 @@ witness_restore(struct lock_object *lock, const char *file, int line)
{
struct lock_list_entry *lock_list;
struct lock_instance *instance;
struct lock_class *class;
KASSERT(!witness_cold, ("%s: witness_cold", __func__));
if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL)
return;
if (lock->lo_class->lc_flags & LC_SLEEPLOCK)
class = LOCK_CLASS(lock);
if (class->lc_flags & LC_SLEEPLOCK)
lock_list = curthread->td_sleeplocks;
else {
if (witness_skipspin)
@ -1847,7 +1853,7 @@ witness_restore(struct lock_object *lock, const char *file, int line)
instance = find_instance(lock_list, lock);
if (instance == NULL)
panic("%s: lock (%s) %s not locked", __func__,
lock->lo_class->lc_name, lock->lo_name);
class->lc_name, lock->lo_name);
lock->lo_witness->w_file = file;
lock->lo_witness->w_line = line;
instance->li_file = file;
@ -1859,23 +1865,25 @@ witness_assert(struct lock_object *lock, int flags, const char *file, int line)
{
#ifdef INVARIANT_SUPPORT
struct lock_instance *instance;
struct lock_class *class;
if (lock->lo_witness == NULL || witness_watch == 0 || panicstr != NULL)
return;
if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) != 0)
class = LOCK_CLASS(lock);
if ((class->lc_flags & LC_SLEEPLOCK) != 0)
instance = find_instance(curthread->td_sleeplocks, lock);
else if ((lock->lo_class->lc_flags & LC_SPINLOCK) != 0)
else if ((class->lc_flags & LC_SPINLOCK) != 0)
instance = find_instance(PCPU_GET(spinlocks), lock);
else {
panic("Lock (%s) %s is not sleep or spin!",
lock->lo_class->lc_name, lock->lo_name);
class->lc_name, lock->lo_name);
}
file = fixup_filename(file);
switch (flags) {
case LA_UNLOCKED:
if (instance != NULL)
panic("Lock (%s) %s locked @ %s:%d.",
lock->lo_class->lc_name, lock->lo_name, file, line);
class->lc_name, lock->lo_name, file, line);
break;
case LA_LOCKED:
case LA_LOCKED | LA_RECURSED:
@ -1888,25 +1896,25 @@ witness_assert(struct lock_object *lock, int flags, const char *file, int line)
case LA_XLOCKED | LA_NOTRECURSED:
if (instance == NULL) {
panic("Lock (%s) %s not locked @ %s:%d.",
lock->lo_class->lc_name, lock->lo_name, file, line);
class->lc_name, lock->lo_name, file, line);
break;
}
if ((flags & LA_XLOCKED) != 0 &&
(instance->li_flags & LI_EXCLUSIVE) == 0)
panic("Lock (%s) %s not exclusively locked @ %s:%d.",
lock->lo_class->lc_name, lock->lo_name, file, line);
class->lc_name, lock->lo_name, file, line);
if ((flags & LA_SLOCKED) != 0 &&
(instance->li_flags & LI_EXCLUSIVE) != 0)
panic("Lock (%s) %s exclusively locked @ %s:%d.",
lock->lo_class->lc_name, lock->lo_name, file, line);
class->lc_name, lock->lo_name, file, line);
if ((flags & LA_RECURSED) != 0 &&
(instance->li_flags & LI_RECURSEMASK) == 0)
panic("Lock (%s) %s not recursed @ %s:%d.",
lock->lo_class->lc_name, lock->lo_name, file, line);
class->lc_name, lock->lo_name, file, line);
if ((flags & LA_NOTRECURSED) != 0 &&
(instance->li_flags & LI_RECURSEMASK) != 0)
panic("Lock (%s) %s recursed @ %s:%d.",
lock->lo_class->lc_name, lock->lo_name, file, line);
class->lc_name, lock->lo_name, file, line);
break;
default:
panic("Invalid lock assertion at %s:%d.", file, line);

1
sys/sys/_lock.h
View File

@ -32,7 +32,6 @@
#define _SYS__LOCK_H_
struct lock_object {
struct lock_class *lo_class;
const char *lo_name; /* Individual lock name. */
const char *lo_type; /* General lock type. */
u_int lo_flags;

22
sys/sys/lock.h
View File

@ -68,6 +68,20 @@ struct lock_class {
#define LO_UPGRADABLE 0x00200000 /* Lock may be upgraded/downgraded. */
#define LO_DUPOK 0x00400000 /* Don't check for duplicate acquires */
#define LO_ENROLLPEND 0x00800000 /* On the pending enroll list. */
#define LO_CLASSMASK 0x0f000000 /* Class index bitmask. */
/*
* Lock classes are statically assigned an index into the gobal lock_classes
* array. Debugging code looks up the lock class for a given lock object
* by indexing the array.
*/
#define LO_CLASSSHIFT 24
#define LO_CLASSINDEX(lock) ((((lock)->lo_flags) & LO_CLASSMASK) >> LO_CLASSSHIFT)
#define LOCK_CLASS(lock) (lock_classes[LO_CLASSINDEX((lock))])
#define LOCK_CLASS_SPIN_MUTEX 0
#define LOCK_CLASS_SLEEP_MUTEX 1
#define LOCK_CLASS_SX 2
#define LOCK_CLASS_MAX LOCK_CLASS_SX
#define LI_RECURSEMASK 0x0000ffff /* Recursion depth of lock instance. */
#define LI_EXCLUSIVE 0x00010000 /* Exclusive lock instance. */
@ -166,21 +180,21 @@ struct lock_list_entry {
#define LOCK_LOG_LOCK(opname, lo, flags, recurse, file, line) do { \
if (LOCK_LOG_TEST((lo), (flags))) \
CTR5(KTR_LOCK, opname " (%s) %s r = %d at %s:%d", \
(lo)->lo_class->lc_name, (lo)->lo_name, \
LOCK_CLASS(lo)->lc_name, (lo)->lo_name, \
(u_int)(recurse), (file), (line)); \
} while (0)
#define LOCK_LOG_TRY(opname, lo, flags, result, file, line) do { \
if (LOCK_LOG_TEST((lo), (flags))) \
CTR5(KTR_LOCK, "TRY_" opname " (%s) %s result=%d at %s:%d",\
(lo)->lo_class->lc_name, (lo)->lo_name, \
LOCK_CLASS(lo)->lc_name, (lo)->lo_name, \
(u_int)(result), (file), (line)); \
} while (0)
#define LOCK_LOG_INIT(lo, flags) do { \
if (LOCK_LOG_TEST((lo), (flags))) \
CTR4(KTR_LOCK, "%s: %p (%s) %s", __func__, (lo), \
(lo)->lo_class->lc_name, (lo)->lo_name); \
LOCK_CLASS(lo)->lc_name, (lo)->lo_name); \
} while (0)
#define LOCK_LOG_DESTROY(lo, flags) LOCK_LOG_INIT(lo, flags)
@ -199,6 +213,8 @@ extern struct lock_class lock_class_mtx_sleep;
extern struct lock_class lock_class_mtx_spin;
extern struct lock_class lock_class_sx;
extern struct lock_class *lock_classes[];
void spinlock_enter(void);
void spinlock_exit(void);
void witness_init(struct lock_object *);