diff --git a/sys/dev/ioat/ioat.c b/sys/dev/ioat/ioat.c index 6ae13e564eb4..e3deed842d83 100644 --- a/sys/dev/ioat/ioat.c +++ b/sys/dev/ioat/ioat.c @@ -376,12 +376,32 @@ ioat_teardown_intr(struct ioat_softc *ioat) static int ioat_start_channel(struct ioat_softc *ioat) { + struct ioat_dma_hw_descriptor *hw_desc; + struct ioat_descriptor *desc; + struct bus_dmadesc *dmadesc; uint64_t status; uint32_t chanerr; int i; ioat_acquire(&ioat->dmaengine); - ioat_null(&ioat->dmaengine, NULL, NULL, 0); + + /* Submit 'NULL' operation manually to avoid quiescing flag */ + desc = ioat_get_ring_entry(ioat, ioat->head); + dmadesc = &desc->bus_dmadesc; + hw_desc = desc->u.dma; + + dmadesc->callback_fn = NULL; + dmadesc->callback_arg = NULL; + + hw_desc->u.control_raw = 0; + hw_desc->u.control_generic.op = IOAT_OP_COPY; + hw_desc->u.control_generic.completion_update = 1; + hw_desc->size = 8; + hw_desc->src_addr = 0; + hw_desc->dest_addr = 0; + hw_desc->u.control.null = 1; + + ioat_submit_single(ioat); ioat_release(&ioat->dmaengine); for (i = 0; i < 100; i++) { @@ -496,6 +516,7 @@ ioat3_attach(device_t device) ioat->head = ioat->hw_head = 0; ioat->tail = 0; ioat->last_seen = 0; + *ioat->comp_update = 0; return (0); } @@ -641,14 +662,24 @@ ioat_process_events(struct ioat_softc *ioat) boolean_t pending; int error; + CTR0(KTR_IOAT, __func__); + mtx_lock(&ioat->cleanup_lock); + /* + * Don't run while the hardware is being reset. Reset is responsible + * for blocking new work and draining & completing existing work, so + * there is nothing to do until new work is queued after reset anyway. + */ + if (ioat->resetting_cleanup) { + mtx_unlock(&ioat->cleanup_lock); + return; + } + completed = 0; comp_update = *ioat->comp_update; status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK; - CTR0(KTR_IOAT, __func__); - if (status == ioat->last_seen) { /* * If we landed in process_events and nothing has been @@ -1643,6 +1674,13 @@ ioat_shrink_timer_callback(void *arg) /* Slowly scale the ring down if idle. */ mtx_lock(&ioat->submit_lock); + + /* Don't run while the hardware is being reset. */ + if (ioat->resetting) { + mtx_unlock(&ioat->submit_lock); + return; + } + order = ioat->ring_size_order; if (ioat->is_resize_pending || order == IOAT_MIN_ORDER) { mtx_unlock(&ioat->submit_lock); @@ -1712,6 +1750,14 @@ ioat_reset_hw(struct ioat_softc *ioat) ioat_drain_locked(ioat); mtx_unlock(IOAT_REFLK); + /* + * Suspend ioat_process_events while the hardware and softc are in an + * indeterminate state. + */ + mtx_lock(&ioat->cleanup_lock); + ioat->resetting_cleanup = TRUE; + mtx_unlock(&ioat->cleanup_lock); + status = ioat_get_chansts(ioat); if (is_ioat_active(status) || is_ioat_idle(status)) ioat_suspend(ioat); @@ -1793,6 +1839,7 @@ ioat_reset_hw(struct ioat_softc *ioat) */ ioat->tail = ioat->head = ioat->hw_head = 0; ioat->last_seen = 0; + *ioat->comp_update = 0; ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); ioat_write_chancmp(ioat, ioat->comp_update_bus_addr); @@ -1800,17 +1847,28 @@ ioat_reset_hw(struct ioat_softc *ioat) error = 0; out: - mtx_lock(IOAT_REFLK); - ioat->resetting = FALSE; - wakeup(&ioat->resetting); - - ioat->quiescing = FALSE; - wakeup(&ioat->quiescing); - mtx_unlock(IOAT_REFLK); + /* + * Resume completions now that ring state is consistent. + * ioat_start_channel will add a pending completion and if we are still + * blocking completions, we may livelock. + */ + mtx_lock(&ioat->cleanup_lock); + ioat->resetting_cleanup = FALSE; + mtx_unlock(&ioat->cleanup_lock); + /* Enqueues a null operation and ensures it completes. */ if (error == 0) error = ioat_start_channel(ioat); + /* Unblock submission of new work */ + mtx_lock(IOAT_REFLK); + ioat->quiescing = FALSE; + wakeup(&ioat->quiescing); + + ioat->resetting = FALSE; + wakeup(&ioat->resetting); + mtx_unlock(IOAT_REFLK); + return (error); } diff --git a/sys/dev/ioat/ioat_internal.h b/sys/dev/ioat/ioat_internal.h index 86a539ec83af..0879a0e99bfd 100644 --- a/sys/dev/ioat/ioat_internal.h +++ b/sys/dev/ioat/ioat_internal.h @@ -491,7 +491,8 @@ struct ioat_softc { boolean_t is_reset_pending; boolean_t is_channel_running; boolean_t intrdelay_supported; - boolean_t resetting; + boolean_t resetting; /* submit_lock */ + boolean_t resetting_cleanup; /* cleanup_lock */ uint32_t head; uint32_t tail;