d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Simplify hang detection by stealing the techniques used in ixl(4) and

Browse Source 
applying them to em(4).

Rely on iterations through the local timer, and the tx queue state to
determine if an actual hang has occurred. Any time a descriptor is used
(packet sent), the tx queue is flagged as busy. Then when txeof runs, it
either clears the flag when all is clean, or resets it to 1 if ANY are
cleaned, if nothing is cleaned it increments the flag.

Local timer simply checks to see if busy ever reaches MAX (10, which
is compile time configurable), and then sets it as HUNG, at that point
there is one more timer cycle in which to have any cleans, if not a
watchdog reset will occur.

Differential Revision:	https://reviews.freebsd.org/D2019
Submitted by:	jfv
Reviewed by:	hiren
Obtained from:	Intel Corporation
MFC after:	2 weeks
Relnotes:	Yes
Sponsored by:	Limelight Networks
This commit is contained in:
Sean Bruno 2015-06-02 18:28:41 +00:00
parent b4df5b00f2
commit b7a728aaba
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=283923
2 changed files with 49 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
sys/dev/e1000/if_em.c
View File

@ -1,6 +1,6 @@
/******************************************************************************
Copyright (c) 2001-2014, Intel Corporation
Copyright (c) 2001-2015, Intel Corporation
All rights reserved.
Redistribution and use in source and binary forms, with or without
@ -943,11 +943,9 @@ em_mq_start_locked(if_t ifp, struct tx_ring *txr, struct mbuf *m)
break;
}
if (enq > 0) {
/* Set the watchdog */
txr->queue_status = EM_QUEUE_WORKING;
txr->watchdog_time = ticks;
}
/* Mark the queue as having work */
if ((enq > 0) && (txr->busy == EM_TX_IDLE))
txr->busy = EM_TX_BUSY;
if (txr->tx_avail < EM_MAX_SCATTER)
em_txeof(txr);
@ -1032,12 +1030,12 @@ em_start_locked(if_t ifp, struct tx_ring *txr)
break;
}
/* Mark the queue as having work */
if (txr->busy == EM_TX_IDLE)
txr->busy = EM_TX_BUSY;
/* Send a copy of the frame to the BPF listener */
if_etherbpfmtap(ifp, m_head);
/* Set timeout in case hardware has problems transmitting. */
txr->watchdog_time = ticks;
txr->queue_status = EM_QUEUE_WORKING;
}
return;
@ -2102,8 +2100,6 @@ em_xmit(struct tx_ring *txr, struct mbuf **m_headp)
*/
tx_buffer = &txr->tx_buffers[first];
tx_buffer->next_eop = last;
/* Update the watchdog time early and often */
txr->watchdog_time = ticks;
/*
* Advance the Transmit Descriptor Tail (TDT), this tells the E1000
@ -2247,15 +2243,16 @@ em_local_timer(void *arg)
** and the HUNG state will be static if set.
*/
for (int i = 0; i < adapter->num_queues; i++, txr++) {
if ((txr->queue_status == EM_QUEUE_HUNG) &&
(adapter->pause_frames == 0))
/* Last cycle a queue was declared hung */
if (txr->busy == EM_TX_HUNG)
goto hung;
if (txr->busy >= EM_TX_MAXTRIES)
txr->busy = EM_TX_HUNG;
/* Schedule a TX tasklet if needed */
if (txr->tx_avail <= EM_MAX_SCATTER)
taskqueue_enqueue(txr->tq, &txr->tx_task);
}
adapter->pause_frames = 0;
callout_reset(&adapter->timer, hz, em_local_timer, adapter);
#ifndef DEVICE_POLLING
/* Trigger an RX interrupt to guarantee mbuf refresh */
@ -2274,7 +2271,6 @@ em_local_timer(void *arg)
txr->me, txr->tx_avail, txr->next_to_clean);
if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
adapter->watchdog_events++;
adapter->pause_frames = 0;
em_init_locked(adapter);
}
@ -2343,9 +2339,9 @@ em_update_link_status(struct adapter *adapter)
if (bootverbose)
device_printf(dev, "Link is Down\n");
adapter->link_active = 0;
/* Link down, disable watchdog */
/* Link down, disable hang detection */
for (int i = 0; i < adapter->num_queues; i++, txr++)
txr->queue_status = EM_QUEUE_IDLE;
txr->busy = EM_TX_IDLE;
if_link_state_change(ifp, LINK_STATE_DOWN);
}
}
@ -2376,10 +2372,10 @@ em_stop(void *arg)
/* Tell the stack that the interface is no longer active */
if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
/* Unarm watchdog timer. */
/* Disarm Hang Detection. */
for (int i = 0; i < adapter->num_queues; i++, txr++) {
EM_TX_LOCK(txr);
txr->queue_status = EM_QUEUE_IDLE;
txr->busy = EM_TX_IDLE;
EM_TX_UNLOCK(txr);
}
@ -3358,7 +3354,7 @@ em_setup_transmit_ring(struct tx_ring *txr)
/* Set number of descriptors available */
txr->tx_avail = adapter->num_tx_desc;
txr->queue_status = EM_QUEUE_IDLE;
txr->busy = EM_TX_IDLE;
/* Clear checksum offload context. */
txr->last_hw_offload = 0;
@ -3419,7 +3415,7 @@ em_initialize_transmit_unit(struct adapter *adapter)
E1000_READ_REG(&adapter->hw, E1000_TDBAL(i)),
E1000_READ_REG(&adapter->hw, E1000_TDLEN(i)));
txr->queue_status = EM_QUEUE_IDLE;
txr->busy = EM_TX_IDLE;
}
/* Set the default values for the Tx Inter Packet Gap timer */
@ -3802,9 +3798,9 @@ em_txeof(struct tx_ring *txr)
return;
#endif /* DEV_NETMAP */
/* No work, make sure watchdog is off */
/* No work, make sure hang detection is disabled */
if (txr->tx_avail == adapter->num_tx_desc) {
txr->queue_status = EM_QUEUE_IDLE;
txr->busy = EM_TX_IDLE;
return;
}
@ -3847,7 +3843,6 @@ em_txeof(struct tx_ring *txr)
tx_buffer->m_head = NULL;
}
tx_buffer->next_eop = -1;
txr->watchdog_time = ticks;
if (++first == adapter->num_tx_desc)
first = 0;
@ -3872,14 +3867,16 @@ em_txeof(struct tx_ring *txr)
txr->next_to_clean = first;
/*
** Watchdog calculation, we know there's
** work outstanding or the first return
** would have been taken, so none processed
** for too long indicates a hang. local timer
** will examine this and do a reset if needed.
** Hang detection: we know there's work outstanding
** or the entry return would have been taken, so no
** descriptor processed here indicates a potential hang.
** The local timer will examine this and do a reset if needed.
*/
if ((!processed) && ((ticks - txr->watchdog_time) > EM_WATCHDOG))
txr->queue_status = EM_QUEUE_HUNG;
if (processed == 0) {
if (txr->busy != EM_TX_HUNG)
++txr->busy;
} else /* At least one descriptor was cleaned */
txr->busy = EM_TX_BUSY; /* note this clears HUNG */
/*
* If we have a minimum free, clear IFF_DRV_OACTIVE
@ -3891,10 +3888,9 @@ em_txeof(struct tx_ring *txr)
if (txr->tx_avail >= EM_MAX_SCATTER)
if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE);
/* Disable watchdog if all clean */
if (txr->tx_avail == adapter->num_tx_desc) {
txr->queue_status = EM_QUEUE_IDLE;
}
/* Disable hang detection if all clean */
if (txr->tx_avail == adapter->num_tx_desc)
txr->busy = EM_TX_IDLE;
}
@ -5130,12 +5126,7 @@ em_update_stats_counters(struct adapter *adapter)
adapter->stats.rlec += E1000_READ_REG(&adapter->hw, E1000_RLEC);
adapter->stats.xonrxc += E1000_READ_REG(&adapter->hw, E1000_XONRXC);
adapter->stats.xontxc += E1000_READ_REG(&adapter->hw, E1000_XONTXC);
/*
** For watchdog management we need to know if we have been
** paused during the last interval, so capture that here.
*/
adapter->pause_frames = E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
adapter->stats.xoffrxc += adapter->pause_frames;
adapter->stats.xoffrxc += E1000_READ_REG(&adapter->hw, E1000_XOFFRXC);
adapter->stats.xofftxc += E1000_READ_REG(&adapter->hw, E1000_XOFFTXC);
adapter->stats.fcruc += E1000_READ_REG(&adapter->hw, E1000_FCRUC);
adapter->stats.prc64 += E1000_READ_REG(&adapter->hw, E1000_PRC64);
@ -5762,7 +5753,7 @@ em_print_debug_info(struct adapter *adapter)
device_printf(dev, "hw rdh = %d, hw rdt = %d\n",
E1000_READ_REG(&adapter->hw, E1000_RDH(0)),
E1000_READ_REG(&adapter->hw, E1000_RDT(0)));
device_printf(dev, "Tx Queue Status = %d\n", txr->queue_status);
device_printf(dev, "Tx Queue Status = %d\n", txr->busy);
device_printf(dev, "TX descriptors avail = %d\n",
txr->tx_avail);
device_printf(dev, "Tx Descriptors avail failure = %ld\n",

22
sys/dev/e1000/if_em.h
View File

@ -1,6 +1,6 @@
/******************************************************************************
Copyright (c) 2001-2011, Intel Corporation
Copyright (c) 2001-2015, Intel Corporation
All rights reserved.
Redistribution and use in source and binary forms, with or without
@ -188,9 +188,19 @@
#define EM_EEPROM_APME 0x400;
#define EM_82544_APME 0x0004;
#define EM_QUEUE_IDLE 0
#define EM_QUEUE_WORKING 1
#define EM_QUEUE_HUNG 2
/*
* Driver state logic for the detection of a hung state
* in hardware. Set TX_HUNG whenever a TX packet is used
* (data is sent) and clear it when txeof() is invoked if
* any descriptors from the ring are cleaned/reclaimed.
* Increment internal counter if no descriptors are cleaned
* and compare to TX_MAXTRIES. When counter > TX_MAXTRIES,
* reset adapter.
*/
#define EM_TX_IDLE 0x00000000
#define EM_TX_BUSY 0x00000001
#define EM_TX_HUNG 0x80000000
#define EM_TX_MAXTRIES 10
/*
* TDBA/RDBA should be aligned on 16 byte boundary. But TDLEN/RDLEN should be
@ -281,8 +291,7 @@ struct tx_ring {
u32 me;
u32 msix;
u32 ims;
int queue_status;
int watchdog_time;
int busy;
struct em_dma_alloc txdma;
struct e1000_tx_desc *tx_base;
struct task tx_task;
@ -368,7 +377,6 @@ struct adapter {
int if_flags;
int max_frame_size;
int min_frame_size;
int pause_frames;
struct mtx core_mtx;
int em_insert_vlan_header;
u32 ims;