ioat(4): Don't "complete" DMA descriptors prematurely
In r304602, I mistakenly removed the ioat_process_events check that we weren't processing events before the hardware had completed the descriptor ("last_seen"). Reinstate that logic. Keep the defensive loop condition and additionally make sure we've actually completed a descriptor before blindly chasing the ring around. In reset, queue and finish the startup command before allowing any event processing or submission to occur. Avoid potential missed callouts by requeueing the poll later.
This commit is contained in:
parent
704fa4fd26
commit
1bbc06b85b
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=305027
@ -683,7 +683,16 @@ ioat_process_events(struct ioat_softc *ioat)
|
||||
__func__, ioat->chan_idx, comp_update, ioat->last_seen);
|
||||
status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK;
|
||||
|
||||
while (ioat_get_active(ioat) > 0) {
|
||||
if (status == ioat->last_seen) {
|
||||
/*
|
||||
* If we landed in process_events and nothing has been
|
||||
* completed, check for a timeout due to channel halt.
|
||||
*/
|
||||
goto out;
|
||||
}
|
||||
|
||||
desc = ioat_get_ring_entry(ioat, ioat->tail - 1);
|
||||
while (desc->hw_desc_bus_addr != status && ioat_get_active(ioat) > 0) {
|
||||
desc = ioat_get_ring_entry(ioat, ioat->tail);
|
||||
dmadesc = &desc->bus_dmadesc;
|
||||
CTR4(KTR_IOAT, "channel=%u completing desc %u ok cb %p(%p)",
|
||||
@ -695,8 +704,6 @@ ioat_process_events(struct ioat_softc *ioat)
|
||||
|
||||
completed++;
|
||||
ioat->tail++;
|
||||
if (desc->hw_desc_bus_addr == status)
|
||||
break;
|
||||
}
|
||||
|
||||
if (completed != 0) {
|
||||
@ -704,6 +711,7 @@ ioat_process_events(struct ioat_softc *ioat)
|
||||
ioat->stats.descriptors_processed += completed;
|
||||
}
|
||||
|
||||
out:
|
||||
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
|
||||
|
||||
/* Perform a racy check first; only take the locks if it passes. */
|
||||
@ -1913,19 +1921,17 @@ ioat_reset_hw(struct ioat_softc *ioat)
|
||||
error = 0;
|
||||
|
||||
out:
|
||||
/* Enqueues a null operation and ensures it completes. */
|
||||
if (error == 0)
|
||||
error = ioat_start_channel(ioat);
|
||||
|
||||
/*
|
||||
* Resume completions now that ring state is consistent.
|
||||
* ioat_start_channel will add a pending completion and if we are still
|
||||
* blocking completions, we may livelock.
|
||||
*/
|
||||
mtx_lock(&ioat->cleanup_lock);
|
||||
ioat->resetting_cleanup = FALSE;
|
||||
mtx_unlock(&ioat->cleanup_lock);
|
||||
|
||||
/* Enqueues a null operation and ensures it completes. */
|
||||
if (error == 0)
|
||||
error = ioat_start_channel(ioat);
|
||||
|
||||
/* Unblock submission of new work */
|
||||
mtx_lock(IOAT_REFLK);
|
||||
ioat->quiescing = FALSE;
|
||||
@ -1933,6 +1939,10 @@ ioat_reset_hw(struct ioat_softc *ioat)
|
||||
|
||||
ioat->resetting = FALSE;
|
||||
wakeup(&ioat->resetting);
|
||||
|
||||
if (ioat->is_completion_pending)
|
||||
callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback,
|
||||
ioat);
|
||||
mtx_unlock(IOAT_REFLK);
|
||||
|
||||
return (error);
|
||||
|
Loading…
Reference in New Issue
Block a user