From 0eb8d462321b9b88bc4dd2fe3e1aa5fc8f460bb6 Mon Sep 17 00:00:00 2001 From: Hans Petter Selasky Date: Wed, 14 Sep 2016 12:07:34 +0000 Subject: [PATCH] Improve USB polling mode by not locking any mutexes, asserting any mutexes or using any callouts when active. Trying to lock a mutex when KDB is active or the scheduler is stopped can result in infinite wait loops. The same goes for calling callout related functions which in turn lock mutexes. If the USB controller at which a USB keyboard is connected is idle when KDB is entered, polling the USB keyboard via USB will always succeed. Else polling may fail depending on which state the USB subsystem and USB interrupt handler is in. This is unavoidable unless KDB can wait for USB interrupt threads to complete before stalling the CPU(s). Tested by: Bruce Evans MFC after: 4 weeks --- sys/dev/usb/input/ukbd.c | 63 +++++++++------------------- sys/dev/usb/quirk/usb_quirk.c | 30 +++++++------- sys/dev/usb/serial/usb_serial.c | 5 +-- sys/dev/usb/serial/usb_serial.h | 6 +-- sys/dev/usb/storage/umass.c | 2 +- sys/dev/usb/usb_busdma.c | 14 +++---- sys/dev/usb/usb_core.c | 14 +++++++ sys/dev/usb/usb_core.h | 23 ++++++----- sys/dev/usb/usb_dev.c | 6 +-- sys/dev/usb/usb_device.c | 4 +- sys/dev/usb/usb_freebsd.h | 3 ++ sys/dev/usb/usb_freebsd_loader.h | 3 ++ sys/dev/usb/usb_generic.c | 4 +- sys/dev/usb/usb_hub.c | 8 ++-- sys/dev/usb/usb_msctest.c | 8 ++-- sys/dev/usb/usb_process.c | 23 ++++++----- sys/dev/usb/usb_request.c | 6 +-- sys/dev/usb/usb_transfer.c | 70 ++++++++++++++++++++------------ sys/dev/usb/usbdi.h | 57 +++++++++++++++++++++++++- 19 files changed, 208 insertions(+), 141 deletions(-) diff --git a/sys/dev/usb/input/ukbd.c b/sys/dev/usb/input/ukbd.c index 331e097db40a..7631324c8ede 100644 --- a/sys/dev/usb/input/ukbd.c +++ b/sys/dev/usb/input/ukbd.c @@ -60,8 +60,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include -#include #include #include @@ -248,32 +246,9 @@ struct ukbd_softc { SCAN_PREFIX_CTL | SCAN_PREFIX_SHIFT) #define SCAN_CHAR(c) ((c) & 0x7f) -#define UKBD_LOCK() mtx_lock(&Giant) -#define UKBD_UNLOCK() mtx_unlock(&Giant) - -#ifdef INVARIANTS - -/* - * Assert that the lock is held in all contexts - * where the code can be executed. - */ -#define UKBD_LOCK_ASSERT() mtx_assert(&Giant, MA_OWNED) - -/* - * Assert that the lock is held in the contexts - * where it really has to be so. - */ -#define UKBD_CTX_LOCK_ASSERT() \ - do { \ - if (!kdb_active && panicstr == NULL) \ - mtx_assert(&Giant, MA_OWNED); \ - } while (0) -#else - -#define UKBD_LOCK_ASSERT() (void)0 -#define UKBD_CTX_LOCK_ASSERT() (void)0 - -#endif +#define UKBD_LOCK() USB_MTX_LOCK(&Giant) +#define UKBD_UNLOCK() USB_MTX_UNLOCK(&Giant) +#define UKBD_LOCK_ASSERT() USB_MTX_ASSERT(&Giant, MA_OWNED) struct ukbd_mods { uint32_t mask, key; @@ -400,7 +375,7 @@ ukbd_start_timer(struct ukbd_softc *sc) sc->sc_co_basetime += delay; /* This is rarely called, so prefer precision to efficiency. */ prec = qmin(delay >> 7, SBT_1MS * 10); - callout_reset_sbt(&sc->sc_callout.co, sc->sc_co_basetime, prec, + usb_callout_reset_sbt(&sc->sc_callout, sc->sc_co_basetime, prec, ukbd_timeout, sc, C_ABSOLUTE); } @@ -408,7 +383,7 @@ static void ukbd_put_key(struct ukbd_softc *sc, uint32_t key) { - UKBD_CTX_LOCK_ASSERT(); + UKBD_LOCK_ASSERT(); DPRINTF("0x%02x (%d) %s\n", key, key, (key & KEY_RELEASE) ? "released" : "pressed"); @@ -429,12 +404,12 @@ static void ukbd_do_poll(struct ukbd_softc *sc, uint8_t wait) { - UKBD_CTX_LOCK_ASSERT(); + UKBD_LOCK_ASSERT(); KASSERT((sc->sc_flags & UKBD_FLAG_POLLING) != 0, ("ukbd_do_poll called when not polling\n")); DPRINTFN(2, "polling\n"); - if (!kdb_active && !SCHEDULER_STOPPED()) { + if (USB_IN_POLLING_MODE_FUNC() == 0) { /* * In this context the kernel is polling for input, * but the USB subsystem works in normal interrupt-driven @@ -479,9 +454,9 @@ ukbd_get_key(struct ukbd_softc *sc, uint8_t wait) { int32_t c; - UKBD_CTX_LOCK_ASSERT(); - KASSERT((!kdb_active && !SCHEDULER_STOPPED()) - || (sc->sc_flags & UKBD_FLAG_POLLING) != 0, + UKBD_LOCK_ASSERT(); + KASSERT((USB_IN_POLLING_MODE_FUNC() == 0) || + (sc->sc_flags & UKBD_FLAG_POLLING) != 0, ("not polling in kdb or panic\n")); if (sc->sc_inputs == 0 && @@ -519,7 +494,7 @@ ukbd_interrupt(struct ukbd_softc *sc) uint8_t i; uint8_t j; - UKBD_CTX_LOCK_ASSERT(); + UKBD_LOCK_ASSERT(); if (sc->sc_ndata.keycode[0] == KEY_ERROR) return; @@ -615,7 +590,7 @@ ukbd_event_keyinput(struct ukbd_softc *sc) { int c; - UKBD_CTX_LOCK_ASSERT(); + UKBD_LOCK_ASSERT(); if ((sc->sc_flags & UKBD_FLAG_POLLING) != 0) return; @@ -838,7 +813,7 @@ ukbd_intr_callback(struct usb_xfer *xfer, usb_error_t error) ukbd_interrupt(sc); - if (ukbd_any_key_pressed(sc)) { + if (ukbd_any_key_pressed(sc) != 0) { ukbd_start_timer(sc); } @@ -1507,7 +1482,7 @@ ukbd_check(keyboard_t *kbd) { struct ukbd_softc *sc = kbd->kb_data; - UKBD_CTX_LOCK_ASSERT(); + UKBD_LOCK_ASSERT(); if (!KBD_IS_ACTIVE(kbd)) return (0); @@ -1532,7 +1507,7 @@ ukbd_check_char_locked(keyboard_t *kbd) { struct ukbd_softc *sc = kbd->kb_data; - UKBD_CTX_LOCK_ASSERT(); + UKBD_LOCK_ASSERT(); if (!KBD_IS_ACTIVE(kbd)) return (0); @@ -1569,7 +1544,7 @@ ukbd_read(keyboard_t *kbd, int wait) #endif - UKBD_CTX_LOCK_ASSERT(); + UKBD_LOCK_ASSERT(); if (!KBD_IS_ACTIVE(kbd)) return (-1); @@ -1618,7 +1593,7 @@ ukbd_read_char_locked(keyboard_t *kbd, int wait) uint32_t scancode; #endif - UKBD_CTX_LOCK_ASSERT(); + UKBD_LOCK_ASSERT(); if (!KBD_IS_ACTIVE(kbd)) return (NOKEY); @@ -1962,7 +1937,7 @@ ukbd_ioctl(keyboard_t *kbd, u_long cmd, caddr_t arg) case KDGKBSTATE: case KDSKBSTATE: case KDSETLED: - if (!mtx_owned(&Giant) && !SCHEDULER_STOPPED()) + if (!mtx_owned(&Giant) && !USB_IN_POLLING_MODE_FUNC()) return (EDEADLK); /* best I could come up with */ /* FALLTHROUGH */ default: @@ -1980,7 +1955,7 @@ ukbd_clear_state(keyboard_t *kbd) { struct ukbd_softc *sc = kbd->kb_data; - UKBD_CTX_LOCK_ASSERT(); + UKBD_LOCK_ASSERT(); sc->sc_flags &= ~(UKBD_FLAG_COMPOSE | UKBD_FLAG_POLLING); sc->sc_state &= LOCK_MASK; /* preserve locking key state */ diff --git a/sys/dev/usb/quirk/usb_quirk.c b/sys/dev/usb/quirk/usb_quirk.c index efc72204e9eb..1c253585f40c 100644 --- a/sys/dev/usb/quirk/usb_quirk.c +++ b/sys/dev/usb/quirk/usb_quirk.c @@ -659,7 +659,7 @@ usb_test_quirk_by_info(const struct usbd_lookup_info *info, uint16_t quirk) if (quirk == UQ_NONE) goto done; - mtx_lock(&usb_quirk_mtx); + USB_MTX_LOCK(&usb_quirk_mtx); for (x = 0; x != USB_DEV_QUIRKS_MAX; x++) { /* see if quirk information does not match */ @@ -683,13 +683,13 @@ usb_test_quirk_by_info(const struct usbd_lookup_info *info, uint16_t quirk) /* lookup quirk */ for (y = 0; y != USB_SUB_QUIRKS_MAX; y++) { if (usb_quirks[x].quirks[y] == quirk) { - mtx_unlock(&usb_quirk_mtx); + USB_MTX_UNLOCK(&usb_quirk_mtx); DPRINTF("Found quirk '%s'.\n", usb_quirkstr(quirk)); return (1); } } } - mtx_unlock(&usb_quirk_mtx); + USB_MTX_UNLOCK(&usb_quirk_mtx); done: return (0); /* no quirk match */ } @@ -700,7 +700,7 @@ usb_quirk_get_entry(uint16_t vid, uint16_t pid, { uint16_t x; - mtx_assert(&usb_quirk_mtx, MA_OWNED); + USB_MTX_ASSERT(&usb_quirk_mtx, MA_OWNED); if ((vid | pid | lo_rev | hi_rev) == 0) { /* all zero - special case */ @@ -768,7 +768,7 @@ usb_quirk_ioctl(unsigned long cmd, caddr_t data, if (y >= USB_DEV_QUIRKS_MAX) { return (EINVAL); } - mtx_lock(&usb_quirk_mtx); + USB_MTX_LOCK(&usb_quirk_mtx); /* copy out data */ pgq->vid = usb_quirks[y].vid; pgq->pid = usb_quirks[y].pid; @@ -777,7 +777,7 @@ usb_quirk_ioctl(unsigned long cmd, caddr_t data, strlcpy(pgq->quirkname, usb_quirkstr(usb_quirks[y].quirks[x]), sizeof(pgq->quirkname)); - mtx_unlock(&usb_quirk_mtx); + USB_MTX_UNLOCK(&usb_quirk_mtx); return (0); /* success */ case USB_QUIRK_NAME_GET: @@ -810,11 +810,11 @@ usb_quirk_ioctl(unsigned long cmd, caddr_t data, if (y == UQ_NONE) { return (EINVAL); } - mtx_lock(&usb_quirk_mtx); + USB_MTX_LOCK(&usb_quirk_mtx); pqe = usb_quirk_get_entry(pgq->vid, pgq->pid, pgq->bcdDeviceLow, pgq->bcdDeviceHigh, 1); if (pqe == NULL) { - mtx_unlock(&usb_quirk_mtx); + USB_MTX_UNLOCK(&usb_quirk_mtx); return (EINVAL); } for (x = 0; x != USB_SUB_QUIRKS_MAX; x++) { @@ -823,7 +823,7 @@ usb_quirk_ioctl(unsigned long cmd, caddr_t data, break; } } - mtx_unlock(&usb_quirk_mtx); + USB_MTX_UNLOCK(&usb_quirk_mtx); if (x == USB_SUB_QUIRKS_MAX) { return (ENOMEM); } @@ -848,11 +848,11 @@ usb_quirk_ioctl(unsigned long cmd, caddr_t data, if (y == UQ_NONE) { return (EINVAL); } - mtx_lock(&usb_quirk_mtx); + USB_MTX_LOCK(&usb_quirk_mtx); pqe = usb_quirk_get_entry(pgq->vid, pgq->pid, pgq->bcdDeviceLow, pgq->bcdDeviceHigh, 0); if (pqe == NULL) { - mtx_unlock(&usb_quirk_mtx); + USB_MTX_UNLOCK(&usb_quirk_mtx); return (EINVAL); } for (x = 0; x != USB_SUB_QUIRKS_MAX; x++) { @@ -862,7 +862,7 @@ usb_quirk_ioctl(unsigned long cmd, caddr_t data, } } if (x == USB_SUB_QUIRKS_MAX) { - mtx_unlock(&usb_quirk_mtx); + USB_MTX_UNLOCK(&usb_quirk_mtx); return (ENOMEM); } for (x = 0; x != USB_SUB_QUIRKS_MAX; x++) { @@ -874,7 +874,7 @@ usb_quirk_ioctl(unsigned long cmd, caddr_t data, /* all quirk entries are unused - release */ memset(pqe, 0, sizeof(*pqe)); } - mtx_unlock(&usb_quirk_mtx); + USB_MTX_UNLOCK(&usb_quirk_mtx); return (0); /* success */ default: @@ -965,14 +965,14 @@ usb_quirk_add_entry_from_str(const char *name, const char *env) printf("%s: Too many USB quirks, only %d allowed!\n", name, USB_SUB_QUIRKS_MAX); } - mtx_lock(&usb_quirk_mtx); + USB_MTX_LOCK(&usb_quirk_mtx); new = usb_quirk_get_entry(entry.vid, entry.pid, entry.lo_rev, entry.hi_rev, 1); if (new == NULL) printf("%s: USB quirks table is full!\n", name); else memcpy(new->quirks, entry.quirks, sizeof(entry.quirks)); - mtx_unlock(&usb_quirk_mtx); + USB_MTX_UNLOCK(&usb_quirk_mtx); } else { printf("%s: No USB quirks found!\n", name); } diff --git a/sys/dev/usb/serial/usb_serial.c b/sys/dev/usb/serial/usb_serial.c index bdcc10f14509..23ee3305a181 100644 --- a/sys/dev/usb/serial/usb_serial.c +++ b/sys/dev/usb/serial/usb_serial.c @@ -79,7 +79,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include @@ -1575,7 +1574,7 @@ ucom_cngetc(struct consdev *cd) UCOM_MTX_UNLOCK(sc); /* poll if necessary */ - if (kdb_active && sc->sc_callback->ucom_poll) + if (USB_IN_POLLING_MODE_FUNC() && sc->sc_callback->ucom_poll) (sc->sc_callback->ucom_poll) (sc); return (c); @@ -1611,7 +1610,7 @@ ucom_cnputc(struct consdev *cd, int c) UCOM_MTX_UNLOCK(sc); /* poll if necessary */ - if (kdb_active && sc->sc_callback->ucom_poll) { + if (USB_IN_POLLING_MODE_FUNC() && sc->sc_callback->ucom_poll) { (sc->sc_callback->ucom_poll) (sc); /* simple flow control */ if (temp == 0) diff --git a/sys/dev/usb/serial/usb_serial.h b/sys/dev/usb/serial/usb_serial.h index d003bf1748c0..9fbd373b0736 100644 --- a/sys/dev/usb/serial/usb_serial.h +++ b/sys/dev/usb/serial/usb_serial.h @@ -194,9 +194,9 @@ struct ucom_softc { uint8_t sc_jitterbuf[UCOM_JITTERBUF_SIZE]; }; -#define UCOM_MTX_ASSERT(sc, what) mtx_assert((sc)->sc_mtx, what) -#define UCOM_MTX_LOCK(sc) mtx_lock((sc)->sc_mtx) -#define UCOM_MTX_UNLOCK(sc) mtx_unlock((sc)->sc_mtx) +#define UCOM_MTX_ASSERT(sc, what) USB_MTX_ASSERT((sc)->sc_mtx, what) +#define UCOM_MTX_LOCK(sc) USB_MTX_LOCK((sc)->sc_mtx) +#define UCOM_MTX_UNLOCK(sc) USB_MTX_UNLOCK((sc)->sc_mtx) #define UCOM_UNLOAD_DRAIN(x) \ SYSUNINIT(var, SI_SUB_KLD - 2, SI_ORDER_ANY, ucom_drain_all, 0) diff --git a/sys/dev/usb/storage/umass.c b/sys/dev/usb/storage/umass.c index acec1c9da528..0f7a5713dee9 100644 --- a/sys/dev/usb/storage/umass.c +++ b/sys/dev/usb/storage/umass.c @@ -1141,7 +1141,7 @@ umass_cancel_ccb(struct umass_softc *sc) { union ccb *ccb; - mtx_assert(&sc->sc_mtx, MA_OWNED); + USB_MTX_ASSERT(&sc->sc_mtx, MA_OWNED); ccb = sc->sc_transfer.ccb; sc->sc_transfer.ccb = NULL; diff --git a/sys/dev/usb/usb_busdma.c b/sys/dev/usb/usb_busdma.c index a52a9098e46b..dc681aa7a75a 100644 --- a/sys/dev/usb/usb_busdma.c +++ b/sys/dev/usb/usb_busdma.c @@ -508,7 +508,7 @@ usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, done: owned = mtx_owned(uptag->mtx); if (!owned) - mtx_lock(uptag->mtx); + USB_MTX_LOCK(uptag->mtx); uptag->dma_error = (error ? 1 : 0); if (isload) { @@ -517,7 +517,7 @@ usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, cv_broadcast(uptag->cv); } if (!owned) - mtx_unlock(uptag->mtx); + USB_MTX_UNLOCK(uptag->mtx); } /*------------------------------------------------------------------------* @@ -592,7 +592,7 @@ usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg, pc->tag = utag->tag; pc->ismultiseg = (align == 1); - mtx_lock(uptag->mtx); + USB_MTX_LOCK(uptag->mtx); /* load memory into DMA */ err = bus_dmamap_load( @@ -603,7 +603,7 @@ usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg, cv_wait(uptag->cv, uptag->mtx); err = 0; } - mtx_unlock(uptag->mtx); + USB_MTX_UNLOCK(uptag->mtx); if (err || uptag->dma_error) { bus_dmamem_free(utag->tag, ptr, map); @@ -659,7 +659,7 @@ usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync) pc->page_offset_end = size; pc->ismultiseg = 1; - mtx_assert(pc->tag_parent->mtx, MA_OWNED); + USB_MTX_ASSERT(pc->tag_parent->mtx, MA_OWNED); if (size > 0) { if (sync) { @@ -917,7 +917,7 @@ usb_bdma_work_loop(struct usb_xfer_queue *pq) xfer = pq->curr; info = xfer->xroot; - mtx_assert(info->xfer_mtx, MA_OWNED); + USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED); if (xfer->error) { /* some error happened */ @@ -1041,7 +1041,7 @@ usb_bdma_done_event(struct usb_dma_parent_tag *udpt) info = USB_DMATAG_TO_XROOT(udpt); - mtx_assert(info->xfer_mtx, MA_OWNED); + USB_MTX_ASSERT(info->xfer_mtx, MA_OWNED); /* copy error */ info->dma_error = udpt->dma_error; diff --git a/sys/dev/usb/usb_core.c b/sys/dev/usb/usb_core.c index 1516e77425df..bb0616a1558b 100644 --- a/sys/dev/usb/usb_core.c +++ b/sys/dev/usb/usb_core.c @@ -51,6 +51,8 @@ #include #include #include +#include +#include #include #include @@ -64,4 +66,16 @@ const struct usb_string_lang usb_string_lang_en = { MALLOC_DEFINE(M_USB, "USB", "USB"); MALLOC_DEFINE(M_USBDEV, "USBdev", "USB device"); +int +usbd_in_polling_mode(void) +{ + return (USB_IN_POLLING_MODE_VALUE()); +} + +void +usbd_dummy_timeout(void *arg) +{ + /* NOP */ +} + MODULE_VERSION(usb, 1); diff --git a/sys/dev/usb/usb_core.h b/sys/dev/usb/usb_core.h index 739a0039160d..9f52e65af2c4 100644 --- a/sys/dev/usb/usb_core.h +++ b/sys/dev/usb/usb_core.h @@ -39,17 +39,20 @@ USB_MODE_DEVICE ? (((xfer)->endpointno & UE_DIR_IN) ? 0 : 1) : \ (((xfer)->endpointno & UE_DIR_IN) ? 1 : 0)) -/* macros */ +/* locking wrappers for BUS lock */ +#define USB_BUS_LOCK(_b) USB_MTX_LOCK(&(_b)->bus_mtx) +#define USB_BUS_UNLOCK(_b) USB_MTX_UNLOCK(&(_b)->bus_mtx) +#define USB_BUS_LOCK_ASSERT(_b, _t) USB_MTX_ASSERT(&(_b)->bus_mtx, _t) -#define USB_BUS_LOCK(_b) mtx_lock(&(_b)->bus_mtx) -#define USB_BUS_UNLOCK(_b) mtx_unlock(&(_b)->bus_mtx) -#define USB_BUS_LOCK_ASSERT(_b, _t) mtx_assert(&(_b)->bus_mtx, _t) -#define USB_BUS_SPIN_LOCK(_b) mtx_lock_spin(&(_b)->bus_spin_lock) -#define USB_BUS_SPIN_UNLOCK(_b) mtx_unlock_spin(&(_b)->bus_spin_lock) -#define USB_BUS_SPIN_LOCK_ASSERT(_b, _t) mtx_assert(&(_b)->bus_spin_lock, _t) -#define USB_XFER_LOCK(_x) mtx_lock((_x)->xroot->xfer_mtx) -#define USB_XFER_UNLOCK(_x) mtx_unlock((_x)->xroot->xfer_mtx) -#define USB_XFER_LOCK_ASSERT(_x, _t) mtx_assert((_x)->xroot->xfer_mtx, _t) +/* locking wrappers for BUS spin lock */ +#define USB_BUS_SPIN_LOCK(_b) USB_MTX_LOCK_SPIN(&(_b)->bus_spin_lock) +#define USB_BUS_SPIN_UNLOCK(_b) USB_MTX_UNLOCK_SPIN(&(_b)->bus_spin_lock) +#define USB_BUS_SPIN_LOCK_ASSERT(_b, _t) USB_MTX_ASSERT(&(_b)->bus_spin_lock, _t) + +/* locking wrappers for XFER lock */ +#define USB_XFER_LOCK(_x) USB_MTX_LOCK((_x)->xroot->xfer_mtx) +#define USB_XFER_UNLOCK(_x) USB_MTX_UNLOCK((_x)->xroot->xfer_mtx) +#define USB_XFER_LOCK_ASSERT(_x, _t) USB_MTX_ASSERT((_x)->xroot->xfer_mtx, _t) /* helper for converting pointers to integers */ #define USB_P2U(ptr) \ diff --git a/sys/dev/usb/usb_dev.c b/sys/dev/usb/usb_dev.c index 47ab5a03edd3..f88a1189fe75 100644 --- a/sys/dev/usb/usb_dev.c +++ b/sys/dev/usb/usb_dev.c @@ -1159,7 +1159,7 @@ usb_filter_write(struct knote *kn, long hint) f = kn->kn_hook; - mtx_assert(f->priv_mtx, MA_OWNED); + USB_MTX_ASSERT(f->priv_mtx, MA_OWNED); cpd = f->curr_cpd; if (cpd == NULL) { @@ -1200,7 +1200,7 @@ usb_filter_read(struct knote *kn, long hint) f = kn->kn_hook; - mtx_assert(f->priv_mtx, MA_OWNED); + USB_MTX_ASSERT(f->priv_mtx, MA_OWNED); cpd = f->curr_cpd; if (cpd == NULL) { @@ -1730,7 +1730,7 @@ usb_fifo_wait(struct usb_fifo *f) { int err; - mtx_assert(f->priv_mtx, MA_OWNED); + USB_MTX_ASSERT(f->priv_mtx, MA_OWNED); if (f->flag_iserror) { /* we are gone */ diff --git a/sys/dev/usb/usb_device.c b/sys/dev/usb/usb_device.c index dc0e4379c5ae..62c3f74e7041 100644 --- a/sys/dev/usb/usb_device.c +++ b/sys/dev/usb/usb_device.c @@ -1511,13 +1511,13 @@ usbd_clear_stall_proc(struct usb_proc_msg *_pm) /* Change lock */ USB_BUS_UNLOCK(udev->bus); - mtx_lock(&udev->device_mtx); + USB_MTX_LOCK(&udev->device_mtx); /* Start clear stall callback */ usbd_transfer_start(udev->ctrl_xfer[1]); /* Change lock */ - mtx_unlock(&udev->device_mtx); + USB_MTX_UNLOCK(&udev->device_mtx); USB_BUS_LOCK(udev->bus); } diff --git a/sys/dev/usb/usb_freebsd.h b/sys/dev/usb/usb_freebsd.h index 3bc9d2c1eb52..ad0f44708bcc 100644 --- a/sys/dev/usb/usb_freebsd.h +++ b/sys/dev/usb/usb_freebsd.h @@ -90,6 +90,9 @@ #define USB_MAX_AUTO_QUIRK 8 /* maximum number of dynamic quirks */ +#define USB_IN_POLLING_MODE_FUNC() usbd_in_polling_mode() +#define USB_IN_POLLING_MODE_VALUE() (SCHEDULER_STOPPED() || kdb_active) + typedef uint32_t usb_timeout_t; /* milliseconds */ typedef uint32_t usb_frlength_t; /* bytes */ typedef uint32_t usb_frcount_t; /* units */ diff --git a/sys/dev/usb/usb_freebsd_loader.h b/sys/dev/usb/usb_freebsd_loader.h index aa26008bbe40..cc78ebc33016 100644 --- a/sys/dev/usb/usb_freebsd_loader.h +++ b/sys/dev/usb/usb_freebsd_loader.h @@ -85,6 +85,9 @@ #define USB_MAX_AUTO_QUIRK 8 /* maximum number of dynamic quirks */ +#define USB_IN_POLLING_MODE_FUNC() 0 +#define USB_IN_POLLING_MODE_VALUE() 0 + typedef uint32_t usb_timeout_t; /* milliseconds */ typedef uint32_t usb_frlength_t; /* bytes */ typedef uint32_t usb_frcount_t; /* units */ diff --git a/sys/dev/usb/usb_generic.c b/sys/dev/usb/usb_generic.c index 73b62a7ee5e3..f5d876c903c8 100644 --- a/sys/dev/usb/usb_generic.c +++ b/sys/dev/usb/usb_generic.c @@ -236,7 +236,7 @@ ugen_open_pipe_write(struct usb_fifo *f) struct usb_endpoint *ep = usb_fifo_softc(f); struct usb_endpoint_descriptor *ed = ep->edesc; - mtx_assert(f->priv_mtx, MA_OWNED); + USB_MTX_ASSERT(f->priv_mtx, MA_OWNED); if (f->xfer[0] || f->xfer[1]) { /* transfers are already opened */ @@ -305,7 +305,7 @@ ugen_open_pipe_read(struct usb_fifo *f) struct usb_endpoint *ep = usb_fifo_softc(f); struct usb_endpoint_descriptor *ed = ep->edesc; - mtx_assert(f->priv_mtx, MA_OWNED); + USB_MTX_ASSERT(f->priv_mtx, MA_OWNED); if (f->xfer[0] || f->xfer[1]) { /* transfers are already opened */ diff --git a/sys/dev/usb/usb_hub.c b/sys/dev/usb/usb_hub.c index 90a85fc7e833..8479a4e7d8c4 100644 --- a/sys/dev/usb/usb_hub.c +++ b/sys/dev/usb/usb_hub.c @@ -270,11 +270,11 @@ uhub_reset_tt_proc(struct usb_proc_msg *_pm) /* Change lock */ USB_BUS_UNLOCK(udev->bus); - mtx_lock(&sc->sc_mtx); + USB_MTX_LOCK(&sc->sc_mtx); /* Start transfer */ usbd_transfer_start(sc->sc_xfer[UHUB_RESET_TT_TRANSFER]); /* Change lock */ - mtx_unlock(&sc->sc_mtx); + USB_MTX_UNLOCK(&sc->sc_mtx); USB_BUS_LOCK(udev->bus); } #endif @@ -1519,9 +1519,9 @@ uhub_attach(device_t dev) /* Start the interrupt endpoint, if any */ - mtx_lock(&sc->sc_mtx); + USB_MTX_LOCK(&sc->sc_mtx); usbd_transfer_start(sc->sc_xfer[UHUB_INTR_TRANSFER]); - mtx_unlock(&sc->sc_mtx); + USB_MTX_UNLOCK(&sc->sc_mtx); /* Enable automatic power save on all USB HUBs */ diff --git a/sys/dev/usb/usb_msctest.c b/sys/dev/usb/usb_msctest.c index b99fd2918e35..27232ba392aa 100644 --- a/sys/dev/usb/usb_msctest.c +++ b/sys/dev/usb/usb_msctest.c @@ -551,13 +551,13 @@ bbb_command_start(struct bbb_transfer *sc, uint8_t dir, uint8_t lun, memcpy(&sc->cbw->CBWCDB, cmd_ptr, cmd_len); DPRINTFN(1, "SCSI cmd = %*D\n", (int)cmd_len, (char *)sc->cbw->CBWCDB, ":"); - mtx_lock(&sc->mtx); + USB_MTX_LOCK(&sc->mtx); usbd_transfer_start(sc->xfer[sc->state]); while (usbd_transfer_pending(sc->xfer[sc->state])) { cv_wait(&sc->cv, &sc->mtx); } - mtx_unlock(&sc->mtx); + USB_MTX_UNLOCK(&sc->mtx); return (sc->error); } @@ -582,11 +582,11 @@ bbb_raw_write(struct bbb_transfer *sc, const void *data_ptr, size_t data_len, DPRINTFN(1, "BULK DATA = %*D\n", (int)data_len, (const char *)data_ptr, ":"); - mtx_lock(&sc->mtx); + USB_MTX_LOCK(&sc->mtx); usbd_transfer_start(sc->xfer[0]); while (usbd_transfer_pending(sc->xfer[0])) cv_wait(&sc->cv, &sc->mtx); - mtx_unlock(&sc->mtx); + USB_MTX_UNLOCK(&sc->mtx); return (sc->error); } diff --git a/sys/dev/usb/usb_process.c b/sys/dev/usb/usb_process.c index 683c700d2472..ec14cd867045 100644 --- a/sys/dev/usb/usb_process.c +++ b/sys/dev/usb/usb_process.c @@ -115,7 +115,7 @@ usb_process(void *arg) sched_prio(td, up->up_prio); thread_unlock(td); - mtx_lock(up->up_mtx); + USB_MTX_LOCK(up->up_mtx); up->up_curtd = td; @@ -195,7 +195,7 @@ usb_process(void *arg) up->up_ptr = NULL; cv_signal(&up->up_cv); - mtx_unlock(up->up_mtx); + USB_MTX_UNLOCK(up->up_mtx); #if (__FreeBSD_version >= 800000) /* Clear the proc pointer if this is the last thread. */ if (--usb_pcount == 0) @@ -291,11 +291,12 @@ usb_proc_msignal(struct usb_process *up, void *_pm0, void *_pm1) usb_size_t d; uint8_t t; - /* check if gone, return dummy value */ - if (up->up_gone) + /* check if gone or in polling mode, return dummy value */ + if (up->up_gone != 0 || + USB_IN_POLLING_MODE_FUNC() != 0) return (_pm0); - mtx_assert(up->up_mtx, MA_OWNED); + USB_MTX_ASSERT(up->up_mtx, MA_OWNED); t = 0; @@ -376,7 +377,7 @@ usb_proc_is_gone(struct usb_process *up) * structure is initialised. */ if (up->up_mtx != NULL) - mtx_assert(up->up_mtx, MA_OWNED); + USB_MTX_ASSERT(up->up_mtx, MA_OWNED); return (0); } @@ -397,7 +398,7 @@ usb_proc_mwait(struct usb_process *up, void *_pm0, void *_pm1) if (up->up_gone) return; - mtx_assert(up->up_mtx, MA_OWNED); + USB_MTX_ASSERT(up->up_mtx, MA_OWNED); if (up->up_curtd == curthread) { /* Just remove the messages from the queue. */ @@ -437,9 +438,9 @@ usb_proc_drain(struct usb_process *up) return; /* handle special case with Giant */ if (up->up_mtx != &Giant) - mtx_assert(up->up_mtx, MA_NOTOWNED); + USB_MTX_ASSERT(up->up_mtx, MA_NOTOWNED); - mtx_lock(up->up_mtx); + USB_MTX_LOCK(up->up_mtx); /* Set the gone flag */ @@ -472,7 +473,7 @@ usb_proc_drain(struct usb_process *up) DPRINTF("WARNING: Someone is waiting " "for USB process drain!\n"); } - mtx_unlock(up->up_mtx); + USB_MTX_UNLOCK(up->up_mtx); } /*------------------------------------------------------------------------* @@ -493,7 +494,7 @@ usb_proc_rewakeup(struct usb_process *up) if (up->up_gone) return; - mtx_assert(up->up_mtx, MA_OWNED); + USB_MTX_ASSERT(up->up_mtx, MA_OWNED); if (up->up_msleep == 0) { /* re-wakeup */ diff --git a/sys/dev/usb/usb_request.c b/sys/dev/usb/usb_request.c index 7c02282f6538..f84263e977ce 100644 --- a/sys/dev/usb/usb_request.c +++ b/sys/dev/usb/usb_request.c @@ -455,8 +455,8 @@ usbd_do_request_flags(struct usb_device *udev, struct mtx *mtx, return (USB_ERR_INVAL); #endif if ((mtx != NULL) && (mtx != &Giant)) { - mtx_unlock(mtx); - mtx_assert(mtx, MA_NOTOWNED); + USB_MTX_UNLOCK(mtx); + USB_MTX_ASSERT(mtx, MA_NOTOWNED); } /* @@ -710,7 +710,7 @@ usbd_do_request_flags(struct usb_device *udev, struct mtx *mtx, usbd_ctrl_unlock(udev); if ((mtx != NULL) && (mtx != &Giant)) - mtx_lock(mtx); + USB_MTX_LOCK(mtx); switch (err) { case USB_ERR_NORMAL_COMPLETION: diff --git a/sys/dev/usb/usb_transfer.c b/sys/dev/usb/usb_transfer.c index a2ca28b643a7..ad562f1f868e 100644 --- a/sys/dev/usb/usb_transfer.c +++ b/sys/dev/usb/usb_transfer.c @@ -45,7 +45,6 @@ #include #include #include -#include #include #include @@ -329,12 +328,12 @@ usbd_transfer_setup_sub_malloc(struct usb_setup_params *parm, pc->buffer = USB_ADD_BYTES(buf, y * size); pc->page_start = pg; - mtx_lock(pc->tag_parent->mtx); + USB_MTX_LOCK(pc->tag_parent->mtx); if (usb_pc_load_mem(pc, size, 1 /* synchronous */ )) { - mtx_unlock(pc->tag_parent->mtx); + USB_MTX_UNLOCK(pc->tag_parent->mtx); return (1); /* failure */ } - mtx_unlock(pc->tag_parent->mtx); + USB_MTX_UNLOCK(pc->tag_parent->mtx); } } } @@ -2262,14 +2261,14 @@ usb_callback_proc(struct usb_proc_msg *_pm) * We exploit the fact that the mutex is the same for all * callbacks that will be called from this thread: */ - mtx_lock(info->xfer_mtx); + USB_MTX_LOCK(info->xfer_mtx); USB_BUS_LOCK(info->bus); /* Continue where we lost track */ usb_command_wrapper(&info->done_q, info->done_q.curr); - mtx_unlock(info->xfer_mtx); + USB_MTX_UNLOCK(info->xfer_mtx); } /*------------------------------------------------------------------------* @@ -2322,7 +2321,7 @@ usbd_callback_wrapper(struct usb_xfer_queue *pq) USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED); if ((pq->recurse_3 != 0 || mtx_owned(info->xfer_mtx) == 0) && - SCHEDULER_STOPPED() == 0) { + USB_IN_POLLING_MODE_FUNC() == 0) { /* * Cases that end up here: * @@ -3303,7 +3302,9 @@ usbd_transfer_poll(struct usb_xfer **ppxfer, uint16_t max) struct usb_xfer_root *xroot; struct usb_device *udev; struct usb_proc_msg *pm; + struct usb_bus *bus; uint16_t n; + uint16_t drop_bus_spin; uint16_t drop_bus; uint16_t drop_xfer; @@ -3318,36 +3319,47 @@ usbd_transfer_poll(struct usb_xfer **ppxfer, uint16_t max) udev = xroot->udev; if (udev == NULL) continue; /* no USB device */ - if (udev->bus == NULL) + bus = udev->bus; + if (bus == NULL) continue; /* no BUS structure */ - if (udev->bus->methods == NULL) + if (bus->methods == NULL) continue; /* no BUS methods */ - if (udev->bus->methods->xfer_poll == NULL) + if (bus->methods->xfer_poll == NULL) continue; /* no poll method */ - /* make sure that the BUS mutex is not locked */ + drop_bus_spin = 0; drop_bus = 0; - while (mtx_owned(&xroot->udev->bus->bus_mtx) && !SCHEDULER_STOPPED()) { - mtx_unlock(&xroot->udev->bus->bus_mtx); - drop_bus++; - } - - /* make sure that the transfer mutex is not locked */ drop_xfer = 0; - while (mtx_owned(xroot->xfer_mtx) && !SCHEDULER_STOPPED()) { - mtx_unlock(xroot->xfer_mtx); - drop_xfer++; + + if (USB_IN_POLLING_MODE_FUNC() == 0) { + /* make sure that the BUS spin mutex is not locked */ + while (mtx_owned(&bus->bus_spin_lock)) { + mtx_unlock_spin(&bus->bus_spin_lock); + drop_bus_spin++; + } + + /* make sure that the BUS mutex is not locked */ + while (mtx_owned(&bus->bus_mtx)) { + mtx_unlock(&bus->bus_mtx); + drop_bus++; + } + + /* make sure that the transfer mutex is not locked */ + while (mtx_owned(xroot->xfer_mtx)) { + mtx_unlock(xroot->xfer_mtx); + drop_xfer++; + } } /* Make sure cv_signal() and cv_broadcast() is not called */ - USB_BUS_CONTROL_XFER_PROC(udev->bus)->up_msleep = 0; - USB_BUS_EXPLORE_PROC(udev->bus)->up_msleep = 0; - USB_BUS_GIANT_PROC(udev->bus)->up_msleep = 0; - USB_BUS_NON_GIANT_ISOC_PROC(udev->bus)->up_msleep = 0; - USB_BUS_NON_GIANT_BULK_PROC(udev->bus)->up_msleep = 0; + USB_BUS_CONTROL_XFER_PROC(bus)->up_msleep = 0; + USB_BUS_EXPLORE_PROC(bus)->up_msleep = 0; + USB_BUS_GIANT_PROC(bus)->up_msleep = 0; + USB_BUS_NON_GIANT_ISOC_PROC(bus)->up_msleep = 0; + USB_BUS_NON_GIANT_BULK_PROC(bus)->up_msleep = 0; /* poll USB hardware */ - (udev->bus->methods->xfer_poll) (udev->bus); + (bus->methods->xfer_poll) (bus); USB_BUS_LOCK(xroot->bus); @@ -3375,7 +3387,11 @@ usbd_transfer_poll(struct usb_xfer **ppxfer, uint16_t max) /* restore BUS mutex */ while (drop_bus--) - mtx_lock(&xroot->udev->bus->bus_mtx); + mtx_lock(&bus->bus_mtx); + + /* restore BUS spin mutex */ + while (drop_bus_spin--) + mtx_lock_spin(&bus->bus_spin_lock); } } diff --git a/sys/dev/usb/usbdi.h b/sys/dev/usb/usbdi.h index 202ad89fa729..6ad3e43e15f7 100644 --- a/sys/dev/usb/usbdi.h +++ b/sys/dev/usb/usbdi.h @@ -434,6 +434,39 @@ struct usb_attach_arg { #define UAA_DEV_EJECTING 2 }; +/* + * General purpose locking wrappers to ease supporting + * USB polled mode: + */ +#ifdef INVARIANTS +#define USB_MTX_ASSERT(_m, _t) do { \ + if (!USB_IN_POLLING_MODE_FUNC()) \ + mtx_assert(_m, _t); \ +} while (0) +#else +#define USB_MTX_ASSERT(_m, _t) do { } while (0) +#endif + +#define USB_MTX_LOCK(_m) do { \ + if (!USB_IN_POLLING_MODE_FUNC()) \ + mtx_lock(_m); \ +} while (0) + +#define USB_MTX_UNLOCK(_m) do { \ + if (!USB_IN_POLLING_MODE_FUNC()) \ + mtx_unlock(_m); \ +} while (0) + +#define USB_MTX_LOCK_SPIN(_m) do { \ + if (!USB_IN_POLLING_MODE_FUNC()) \ + mtx_lock_spin(_m); \ +} while (0) + +#define USB_MTX_UNLOCK_SPIN(_m) do { \ + if (!USB_IN_POLLING_MODE_FUNC()) \ + mtx_unlock_spin(_m); \ +} while (0) + /* * The following is a wrapper for the callout structure to ease * porting the code to other platforms. @@ -442,8 +475,26 @@ struct usb_callout { struct callout co; }; #define usb_callout_init_mtx(c,m,f) callout_init_mtx(&(c)->co,m,f) -#define usb_callout_reset(c,t,f,d) callout_reset(&(c)->co,t,f,d) -#define usb_callout_stop(c) callout_stop(&(c)->co) +#define usb_callout_reset(c,...) do { \ + if (!USB_IN_POLLING_MODE_FUNC()) \ + callout_reset(&(c)->co, __VA_ARGS__); \ +} while (0) +#define usb_callout_reset_sbt(c,...) do { \ + if (!USB_IN_POLLING_MODE_FUNC()) \ + callout_reset_sbt(&(c)->co, __VA_ARGS__); \ +} while (0) +#define usb_callout_stop(c) do { \ + if (!USB_IN_POLLING_MODE_FUNC()) { \ + callout_stop(&(c)->co); \ + } else { \ + /* \ + * Cannot stop callout when \ + * polling. Set dummy callback \ + * function instead: \ + */ \ + (c)->co.c_func = &usbd_dummy_timeout; \ + } \ +} while (0) #define usb_callout_drain(c) callout_drain(&(c)->co) #define usb_callout_pending(c) callout_pending(&(c)->co) @@ -623,6 +674,8 @@ void usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset, void usbd_start_re_enumerate(struct usb_device *udev); usb_error_t usbd_start_set_config(struct usb_device *, uint8_t); +int usbd_in_polling_mode(void); +void usbd_dummy_timeout(void *); int usb_fifo_attach(struct usb_device *udev, void *priv_sc, struct mtx *priv_mtx, struct usb_fifo_methods *pm,