freebsd-dev/sys/ia64/include/mutex.h
Jason Evans 5704011c0d Reduce userland namespace polution.
#include <proc.h>, since curproc is needed.
2000-10-06 08:11:11 +00:00

569 lines
16 KiB
C

/*-
* Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Berkeley Software Design Inc's name may not be used to endorse or
* promote products derived from this software without specific prior
* written permission.
*
* THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* from BSDI $Id: mutex.h,v 2.7.2.35 2000/04/27 03:10:26 cp Exp $
* $FreeBSD$
*/
#ifndef _MACHINE_MUTEX_H_
#define _MACHINE_MUTEX_H_
#ifndef LOCORE
#include <sys/queue.h>
#ifdef _KERNEL
#include <sys/ktr.h>
#include <sys/proc.h> /* Needed for curproc. */
#include <machine/atomic.h>
#include <machine/cpufunc.h>
#include <machine/globaldata.h>
#include <machine/globals.h>
/*
* If kern_mutex.c is being built, compile non-inlined versions of various
* functions so that kernel modules can use them.
*/
#ifndef _KERN_MUTEX_C_
#define _MTX_INLINE static __inline
#else
#define _MTX_INLINE
#endif
/*
* Mutex flags
*
* Types
*/
#define MTX_DEF 0x1 /* Default (spin/sleep) */
#define MTX_SPIN 0x2 /* Spin only lock */
/* Options */
#define MTX_RLIKELY 0x4 /* (opt) Recursion likely */
#define MTX_NORECURSE 0x8 /* No recursion possible */
#define MTX_NOSPIN 0x10 /* Don't spin before sleeping */
#define MTX_NOSWITCH 0x20 /* Do not switch on release */
#define MTX_FIRST 0x40 /* First spin lock holder */
#define MTX_TOPHALF 0x80 /* Interrupts not disabled on spin */
/* options that should be passed on to mtx_enter_hard, mtx_exit_hard */
#define MTX_HARDOPTS (MTX_DEF | MTX_SPIN | MTX_FIRST | MTX_TOPHALF | MTX_NOSWITCH)
/* Flags/value used in mtx_lock */
#define MTX_RECURSE 0x01 /* (non-spin) lock held recursively */
#define MTX_CONTESTED 0x02 /* (non-spin) lock contested */
#define MTX_FLAGMASK ~(MTX_RECURSE | MTX_CONTESTED)
#define MTX_UNOWNED 0x8 /* Cookie for free mutex */
#endif /* _KERNEL */
/*
* Sleep/spin mutex
*/
struct mtx {
volatile u_int64_t mtx_lock; /* lock owner/gate/flags */
volatile u_int32_t mtx_recurse; /* number of recursive holds */
u_int32_t mtx_savepsr; /* saved psr (for spin locks) */
char *mtx_description;
TAILQ_HEAD(, proc) mtx_blocked;
LIST_ENTRY(mtx) mtx_contested;
struct mtx *mtx_next; /* all locks in system */
struct mtx *mtx_prev;
#ifdef SMP_DEBUG
/* If you add anything here, adjust the mtxf_t definition below */
struct witness *mtx_witness;
LIST_ENTRY(mtx) mtx_held;
const char *mtx_file;
int mtx_line;
#endif /* SMP_DEBUG */
};
/*
* Filler for structs which need to remain the same size
* whether or not SMP_DEBUG is turned on.
*/
typedef struct mtxf {
#ifdef SMP_DEBUG
char mtxf_data[0];
#else
char mtxf_data[4*sizeof(void *) + sizeof(int)];
#endif
} mtxf_t;
#define mp_fixme(string)
#ifdef _KERNEL
/* Misc */
#define CURTHD ((u_int64_t)CURPROC) /* Current thread ID */
/* Prototypes */
void mtx_init(struct mtx *m, char *description, int flag);
void mtx_enter_hard(struct mtx *, int type, int psr);
void mtx_exit_hard(struct mtx *, int type);
void mtx_destroy(struct mtx *m);
/*
* Wrap the following functions with cpp macros so that filenames and line
* numbers are embedded in the code correctly.
*/
#if (defined(KLD_MODULE) || defined(_KERN_MUTEX_C_))
void _mtx_enter(struct mtx *mtxp, int type, const char *file, int line);
int _mtx_try_enter(struct mtx *mtxp, int type, const char *file, int line);
void _mtx_exit(struct mtx *mtxp, int type, const char *file, int line);
#endif
#define mtx_enter(mtxp, type) \
_mtx_enter((mtxp), (type), __FILE__, __LINE__)
#define mtx_try_enter(mtxp, type) \
_mtx_try_enter((mtxp), (type), __FILE__, __LINE__)
#define mtx_exit(mtxp, type) \
_mtx_exit((mtxp), (type), __FILE__, __LINE__)
/* Global locks */
extern struct mtx sched_lock;
extern struct mtx Giant;
/*
* Used to replace return with an exit Giant and return.
*/
#define EGAR(a) \
do { \
mtx_exit(&Giant, MTX_DEF); \
return (a); \
} while (0)
#define VEGAR \
do { \
mtx_exit(&Giant, MTX_DEF); \
return; \
} while (0)
#define DROP_GIANT() \
do { \
int _giantcnt; \
WITNESS_SAVE_DECL(Giant); \
\
WITNESS_SAVE(&Giant, Giant); \
for (_giantcnt = 0; mtx_owned(&Giant); _giantcnt++) \
mtx_exit(&Giant, MTX_DEF)
#define PICKUP_GIANT() \
mtx_assert(&Giant, MA_NOTOWNED); \
while (_giantcnt--) \
mtx_enter(&Giant, MTX_DEF); \
WITNESS_RESTORE(&Giant, Giant); \
} while (0)
#define PARTIAL_PICKUP_GIANT() \
mtx_assert(&Giant, MA_NOTOWNED); \
while (_giantcnt--) \
mtx_enter(&Giant, MTX_DEF); \
WITNESS_RESTORE(&Giant, Giant)
/*
* Debugging
*/
#ifndef SMP_DEBUG
#define mtx_assert(m, what)
#else /* SMP_DEBUG */
#define MA_OWNED 1
#define MA_NOTOWNED 2
#define mtx_assert(m, what) { \
switch ((what)) { \
case MA_OWNED: \
ASS(mtx_owned((m))); \
break; \
case MA_NOTOWNED: \
ASS(!mtx_owned((m))); \
break; \
default: \
panic("unknown mtx_assert at %s:%d", __FILE__, __LINE__); \
} \
}
#ifdef INVARIANTS
#define ASS(ex) MPASS(ex)
#define MPASS(ex) if (!(ex)) panic("Assertion %s failed at %s:%d", \
#ex, __FILE__, __LINE__)
#define MPASS2(ex, what) if (!(ex)) panic("Assertion %s failed at %s:%d", \
what, __FILE__, __LINE__)
#ifdef MTX_STRS
char STR_IEN[] = "psr.i";
char STR_IDIS[] = "!psr.i";
#else /* MTX_STRS */
extern char STR_IEN[];
extern char STR_IDIS[];
#endif /* MTX_STRS */
#define ASS_IEN MPASS2((save_intr() & (1 << 14)), STR_IEN)
#define ASS_IDIS MPASS2(!(save_intr() & (1 << 14)), STR_IDIS)
#endif /* INVARIANTS */
#endif /* SMP_DEBUG */
#if !defined(SMP_DEBUG) || !defined(INVARIANTS)
#define ASS(ex)
#define MPASS(ex)
#define MPASS2(ex, where)
#define ASS_IEN
#define ASS_IDIS
#endif /* !defined(SMP_DEBUG) || !defined(INVARIANTS) */
#ifdef WITNESS
#ifndef SMP_DEBUG
#error WITNESS requires SMP_DEBUG
#endif /* SMP_DEBUG */
#define WITNESS_ENTER(m, t, f, l) \
if ((m)->mtx_witness != NULL) \
witness_enter((m), (t), (f), (l))
#define WITNESS_EXIT(m, t, f, l) \
if ((m)->mtx_witness != NULL) \
witness_exit((m), (t), (f), (l))
#define WITNESS_SLEEP(check, m) witness_sleep(check, (m), __FILE__, __LINE__)
#define WITNESS_SAVE_DECL(n) \
const char * __CONCAT(n, __wf); \
int __CONCAT(n, __wl)
#define WITNESS_SAVE(m, n) \
do { \
if ((m)->mtx_witness != NULL) \
witness_save(m, &__CONCAT(n, __wf), &__CONCAT(n, __wl)); \
} while (0)
#define WITNESS_RESTORE(m, n) \
do { \
if ((m)->mtx_witness != NULL) \
witness_restore(m, __CONCAT(n, __wf), __CONCAT(n, __wl)); \
} while (0)
void witness_init(struct mtx *, int flag);
void witness_destroy(struct mtx *);
void witness_enter(struct mtx *, int, const char *, int);
void witness_try_enter(struct mtx *, int, const char *, int);
void witness_exit(struct mtx *, int, const char *, int);
void witness_display(void(*)(const char *fmt, ...));
void witness_list(struct proc *);
int witness_sleep(int, struct mtx *, const char *, int);
void witness_save(struct mtx *, const char **, int *);
void witness_restore(struct mtx *, const char *, int);
#else /* WITNESS */
#define WITNESS_ENTER(m, t, f, l)
#define WITNESS_EXIT(m, t, f, l)
#define WITNESS_SLEEP(check, m)
#define WITNESS_SAVE_DECL(n)
#define WITNESS_SAVE(m, n)
#define WITNESS_RESTORE(m, n)
/*
* flag++ is slezoid way of shutting up unused parameter warning
* in mtx_init()
*/
#define witness_init(m, flag) flag++
#define witness_destroy(m)
#define witness_enter(m, t, f, l)
#define witness_try_enter(m, t, f, l)
#define witness_exit(m, t, f, l)
#endif /* WITNESS */
/*
* Assembly macros (for internal use only)
*--------------------------------------------------------------------------
*/
/*
* Get a sleep lock, deal with recursion inline
*/
#define _V(x) __STRING(x)
#define _getlock_sleep(mp, tid, type) do { \
if (ia64_cmpxchg_acq_64(&(mp)->mtx_lock, \
MTX_UNOWNED, (tid)) != MTX_UNOWNED) { \
if (((mp)->mtx_lock & MTX_FLAGMASK) != (tid)) \
mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0); \
else { \
atomic_set_64(&(mp)->mtx_lock, MTX_RECURSE); \
(mp)->mtx_recurse++; \
} \
} \
} while (0)
/*
* Get a spin lock, handle recusion inline (as the less common case)
*/
#define _getlock_spin_block(mp, tid, type) do { \
u_int _psr = save_intr(); \
disable_intr(); \
if (ia64_cmpxchg_acq_64(&(mp)->mtx_lock, \
MTX_UNOWNED, (tid)) != MTX_UNOWNED) \
mtx_enter_hard(mp, (type) & MTX_HARDOPTS, _psr); \
else \
(mp)->mtx_savepsr = _psr; \
} while (0)
/*
* Get a lock without any recursion handling. Calls the hard enter
* function if we can't get it inline.
*/
#define _getlock_norecurse(mp, tid, type) do { \
if (ia64_cmpxchg_acq_64(&(mp)->mtx_lock, \
MTX_UNOWNED, (tid)) != MTX_UNOWNED) \
mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0); \
} while (0)
/*
* Release a sleep lock assuming we haven't recursed on it, recursion is
* handled in the hard function.
*/
#define _exitlock_norecurse(mp, tid, type) do { \
if (ia64_cmpxchg_rel_64(&(mp)->mtx_lock, \
(tid), MTX_UNOWNED) != (tid)) \
mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \
} while (0)
/*
* Release a sleep lock when its likely we recursed (the code to
* deal with simple recursion is inline).
*/
#define _exitlock(mp, tid, type) do { \
if (ia64_cmpxchg_rel_64(&(mp)->mtx_lock, \
(tid), MTX_UNOWNED) != (tid)) { \
if (((mp)->mtx_lock & MTX_RECURSE) && \
(--(mp)->mtx_recurse == 0)) \
atomic_clear_64(&(mp)->mtx_lock, MTX_RECURSE); \
else \
mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \
} \
} while (0)
/*
* Release a spin lock (with possible recursion)
*/
#define _exitlock_spin(mp) do { \
if ((mp)->mtx_recurse == 0) { \
int _psr = (mp)->mtx_savepsr; \
ia64_st_rel_64(&(mp)->mtx_lock, MTX_UNOWNED); \
restore_intr(_psr); \
} else { \
(mp)->mtx_recurse--; \
} \
} while (0)
/*
* Externally visible mutex functions
*------------------------------------------------------------------------
*/
/*
* Return non-zero if a mutex is already owned by the current thread
*/
#define mtx_owned(m) (((m)->mtx_lock & MTX_FLAGMASK) == CURTHD)
/* Common strings */
#ifdef MTX_STRS
char STR_mtx_enter_fmt[] = "GOT %s [%p] at %s:%d r=%d";
char STR_mtx_bad_type[] = "((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0";
char STR_mtx_exit_fmt[] = "REL %s [%p] at %s:%d r=%d";
char STR_mtx_owned[] = "mtx_owned(mpp)";
char STR_mtx_recurse[] = "mpp->mtx_recurse == 0";
char STR_mtx_try_enter_fmt[] = "TRY_ENTER %s [%p] at %s:%d result=%d";
#else /* MTX_STRS */
extern char STR_mtx_enter_fmt[];
extern char STR_mtx_bad_type[];
extern char STR_mtx_exit_fmt[];
extern char STR_mtx_owned[];
extern char STR_mtx_recurse[];
extern char STR_mtx_try_enter_fmt[];
#endif /* MTX_STRS */
#ifndef KLD_MODULE
/*
* Get lock 'm', the macro handles the easy (and most common cases) and
* leaves the slow stuff to the mtx_enter_hard() function.
*
* Note: since type is usually a constant much of this code is optimized out
*/
_MTX_INLINE void
_mtx_enter(struct mtx *mtxp, int type, const char *file, int line)
{
struct mtx *mpp = mtxp;
/* bits only valid on mtx_exit() */
MPASS2(((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0,
STR_mtx_bad_type);
if ((type) & MTX_SPIN) {
/*
* Easy cases of spin locks:
*
* 1) We already own the lock and will simply recurse on it (if
* RLIKELY)
*
* 2) The lock is free, we just get it
*/
if ((type) & MTX_RLIKELY) {
/*
* Check for recursion, if we already have this lock we
* just bump the recursion count.
*/
if (mpp->mtx_lock == CURTHD) {
mpp->mtx_recurse++;
goto done;
}
}
if (((type) & MTX_TOPHALF) == 0) {
/*
* If an interrupt thread uses this we must block
* interrupts here.
*/
_getlock_spin_block(mpp, CURTHD, (type) & MTX_HARDOPTS);
} else
_getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
} else {
/* Sleep locks */
if ((type) & MTX_RLIKELY)
_getlock_sleep(mpp, CURTHD, (type) & MTX_HARDOPTS);
else
_getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
}
done:
WITNESS_ENTER(mpp, type, file, line);
CTR5(KTR_LOCK, STR_mtx_enter_fmt,
mpp->mtx_description, mpp, file, line,
mpp->mtx_recurse);
}
/*
* Attempt to get MTX_DEF lock, return non-zero if lock acquired
*
* XXX DOES NOT HANDLE RECURSION
*/
_MTX_INLINE int
_mtx_try_enter(struct mtx *mtxp, int type, const char *file, int line)
{
struct mtx *const mpp = mtxp;
int rval;
rval = atomic_cmpset_64(&mpp->mtx_lock, MTX_UNOWNED, CURTHD);
#ifdef SMP_DEBUG
if (rval && mpp->mtx_witness != NULL) {
ASS(mpp->mtx_recurse == 0);
witness_try_enter(mpp, type, file, line);
}
#endif
CTR5(KTR_LOCK, STR_mtx_try_enter_fmt,
mpp->mtx_description, mpp, file, line, rval);
return rval;
}
/*
* Release lock m
*/
_MTX_INLINE void
_mtx_exit(struct mtx *mtxp, int type, const char *file, int line)
{
struct mtx *const mpp = mtxp;
MPASS2(mtx_owned(mpp), STR_mtx_owned);
WITNESS_EXIT(mpp, type, file, line);
CTR5(KTR_LOCK, STR_mtx_exit_fmt,
mpp->mtx_description, mpp, file, line,
mpp->mtx_recurse);
if ((type) & MTX_SPIN) {
if ((type) & MTX_NORECURSE) {
MPASS2(mpp->mtx_recurse == 0, STR_mtx_recurse);
ia64_st_rel_64(&mpp->mtx_lock, MTX_UNOWNED);
if (((type) & MTX_TOPHALF) == 0)
restore_intr(mpp->mtx_savepsr);
} else
if ((type) & MTX_TOPHALF) {
_exitlock_norecurse(mpp, CURTHD,
(type) & MTX_HARDOPTS);
} else
_exitlock_spin(mpp);
} else {
/* Handle sleep locks */
if ((type) & MTX_RLIKELY) {
_exitlock(mpp, CURTHD, (type) & MTX_HARDOPTS);
} else {
_exitlock_norecurse(mpp, CURTHD,
(type) & MTX_HARDOPTS);
}
}
}
#endif /* KLD_MODULE */
#endif /* _KERNEL */
#else /* !LOCORE */
/*
* Simple assembly macros to get and release non-recursive spin locks
*/
#define MTX_ENTER(lck, rPSR, rOLD, rNEW, rLCK) \
mov rPSR=psr ; \
mov rNEW=globalp ; \
addl rLCK=@ltoff(lck),gp ;; \
ld8 rLCK=[rLCK] ;; \
add rLCK=MTX_LOCK,rLCK ;; \
rsm psr.i ; \
mov ar.ccv=MTX_UNOWNED ; \
add rNEW=PC_CURPROC,rNEW ;; \
ld8 rNEW=[rNEW] ;; \
1: cmpxchg8.acq rOLD=[rLCK],rNEW,ar.ccv ;; \
cmp.eq p1,p0=MTX_UNOWNED,rOLD ;; \
(p1) br.cond.spnt.few 1b ;; \
addl rLCK=@ltoff(lck),gp ;; \
ld8 rLCK=[rLCK] ;; \
add rLCK=MTX_SAVEPSR,rLCK ;; \
st4 [rLCK]=rPSR
#define MTX_EXIT(lck, rTMP, rLCK) \
mov rTMP=MTX_UNOWNED ; \
addl rLCK=@ltoff(lck),gp;; \
ld8 rLCK=[rLCK];; \
add rLCK=MTX_LOCK,rLCK;; \
st8.rel [rLCK]=rTMP,MTX_SAVEPSR-MTX_LOCK ;; \
ld4 rTMP=[rLCK] ;; \
mov psr.l=rTMP ;; \
srlz.d
#endif /* !LOCORE */
#endif /* __MACHINE_MUTEX_H */