diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index 6e6dd5a3d0d2..9ffb31a3e587 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -348,6 +348,11 @@ witness_init(struct lock_object *lock) panic("%s: lock (%s) %s can not be sleepable!\n", __func__, class->lc_name, lock->lo_name); + if ((lock->lo_flags & LO_UPGRADABLE) != 0 && + (class->lc_flags & LC_UPGRADABLE) == 0) + panic("%s: lock (%s) %s can not be upgradable!\n", __func__, + class->lc_name, lock->lo_name); + mtx_lock(&all_mtx); STAILQ_INSERT_TAIL(&all_locks, lock, lo_list); lock->lo_flags |= LO_INITIALIZED; @@ -712,6 +717,72 @@ witness_lock(struct lock_object *lock, int flags, const char *file, int line) curproc->p_pid, lock->lo_name, lle->ll_count - 1); } +void +witness_upgrade(struct lock_object *lock, int flags, const char *file, int line) +{ + struct lock_instance *instance; + struct lock_class *class; + + KASSERT(!witness_cold, ("%s: witness_cold\n", __func__)); + if (lock->lo_witness == NULL || witness_dead || panicstr != NULL) + return; + + class = lock->lo_class; + if ((lock->lo_flags & LO_UPGRADABLE) == 0) + panic("upgrade of non-upgradable lock (%s) %s @ %s:%d", + class->lc_name, lock->lo_name, file, line); + if ((flags & LOP_TRYLOCK) == 0) + panic("non-try upgrade of lock (%s) %s @ %s:%d", class->lc_name, + lock->lo_name, file, line); + if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0) + panic("upgrade of non-sleep lock (%s) %s @ %s:%d", + class->lc_name, lock->lo_name, file, line); + instance = find_instance(curproc->p_sleeplocks, lock); + if (instance == NULL) + panic("upgrade of unlocked lock (%s) %s @ %s:%d", + class->lc_name, lock->lo_name, file, line); + if ((instance->li_flags & LI_EXCLUSIVE) != 0) + panic("upgrade of exclusive lock (%s) %s @ %s:%d", + class->lc_name, lock->lo_name, file, line); + if ((instance->li_flags & LI_RECURSEMASK) != 0) + panic("upgrade of recursed lock (%s) %s r=%d @ %s:%d", + class->lc_name, lock->lo_name, + instance->li_flags & LI_RECURSEMASK, file, line); + instance->li_flags |= LI_EXCLUSIVE; +} + +void +witness_downgrade(struct lock_object *lock, int flags, const char *file, + int line) +{ + struct lock_instance *instance; + struct lock_class *class; + + KASSERT(!witness_cold, ("%s: witness_cold\n", __func__)); + if (lock->lo_witness == NULL || witness_dead || panicstr != NULL) + return; + + class = lock->lo_class; + if ((lock->lo_flags & LO_UPGRADABLE) == 0) + panic("downgrade of non-upgradable lock (%s) %s @ %s:%d", + class->lc_name, lock->lo_name, file, line); + if ((lock->lo_class->lc_flags & LC_SLEEPLOCK) == 0) + panic("downgrade of non-sleep lock (%s) %s @ %s:%d", + class->lc_name, lock->lo_name, file, line); + instance = find_instance(curproc->p_sleeplocks, lock); + if (instance == NULL) + panic("downgrade of unlocked lock (%s) %s @ %s:%d", + class->lc_name, lock->lo_name, file, line); + if ((instance->li_flags & LI_EXCLUSIVE) == 0) + panic("downgrade of shared lock (%s) %s @ %s:%d", + class->lc_name, lock->lo_name, file, line); + if ((instance->li_flags & LI_RECURSEMASK) != 0) + panic("downgrade of recursed lock (%s) %s r=%d @ %s:%d", + class->lc_name, lock->lo_name, + instance->li_flags & LI_RECURSEMASK, file, line); + instance->li_flags &= ~LI_EXCLUSIVE; +} + void witness_unlock(struct lock_object *lock, int flags, const char *file, int line) { diff --git a/sys/sys/lock.h b/sys/sys/lock.h index 8c0f34027a17..e55d9929aa42 100644 --- a/sys/sys/lock.h +++ b/sys/sys/lock.h @@ -60,6 +60,7 @@ struct lock_class { #define LC_SPINLOCK 0x00000002 /* Spin lock. */ #define LC_SLEEPABLE 0x00000004 /* Sleeping allowed with this lock. */ #define LC_RECURSABLE 0x00000008 /* Locks of this type may recurse. */ +#define LC_UPGRADABLE 0x00000010 /* Upgrades and downgrades permitted. */ #define LO_CLASSFLAGS 0x0000ffff /* Class specific flags. */ #define LO_INITIALIZED 0x00010000 /* Lock has been initialized. */ @@ -67,6 +68,7 @@ struct lock_class { #define LO_QUIET 0x00040000 /* Don't log locking operations. */ #define LO_RECURSABLE 0x00080000 /* Lock may recurse. */ #define LO_SLEEPABLE 0x00100000 /* Lock may be held while sleeping. */ +#define LO_UPGRADABLE 0x00200000 /* Lock may be upgraded/downgraded. */ #define LI_RECURSEMASK 0x0000ffff /* Recursion depth of lock instance. */ #define LI_SLEPT 0x00010000 /* Lock instance has been slept with. */ @@ -174,6 +176,8 @@ extern struct lock_class lock_class_sx; void witness_init(struct lock_object *); void witness_destroy(struct lock_object *); void witness_lock(struct lock_object *, int, const char *, int); +void witness_upgrade(struct lock_object *, int, const char *, int); +void witness_downgrade(struct lock_object *, int, const char *, int); void witness_unlock(struct lock_object *, int, const char *, int); void witness_save(struct lock_object *, const char **, int *); void witness_restore(struct lock_object *, const char *, int); @@ -192,6 +196,12 @@ void witness_assert(struct lock_object *, int, const char *, int); #define WITNESS_LOCK(lock, flags, file, line) \ witness_lock((lock), (flags), (file), (line)) +#define WITNESS_UPGRADE(lock, flags, file, line) \ + witness_upgrade((lock), (flags), (file), (line)) + +#define WITNESS_DOWNGRADE(lock, flags, file, line) \ + witness_downgrade((lock), (flags), (file), (line)) + #define WITNESS_UNLOCK(lock, flags, file, line) \ witness_unlock((lock), (flags), (file), (line)) @@ -212,6 +222,8 @@ void witness_assert(struct lock_object *, int, const char *, int); #define WITNESS_INIT(lock) (lock)->lo_flags |= LO_INITIALIZED #define WITNESS_DESTROY(lock) (lock)->lo_flags &= ~LO_INITIALIZED #define WITNESS_LOCK(lock, flags, file, line) +#define WITNESS_UPGRADE(lock, flags, file, line) +#define WITNESS_DOWNGRADE(lock, flags, file, line) #define WITNESS_UNLOCK(lock, flags, file, line) #define WITNESS_SLEEP(check, lock) #define WITNESS_SAVE_DECL(n)