d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
6507fb6f0a
numam-dpdk/drivers/net/cxgbe/sge.c
Thomas Monjalon 5e046832f1 ethdev: rename memzones allocated for DMA 
The helper rte_eth_dma_zone_reserve() is called by PMDs
when probing a new port.
It creates a new memzone with an unique name.
The name of this memzone was using the name of the driver
doing the probe.

In order to avoid assigning the driver before the end of the probing,
the driver name is removed from these memzone names.
The ethdev name (data->name) is not used because it may be too long
and may be not set at this stage of probing.

Syntax of old name: <driver>_<ring>_<port>_<queue>
Syntax of new name: eth_p<port>_q<queue>_<ring>

Signed-off-by: Thomas Monjalon <thomas@monjalon.net>
Tested-by: Andrew Rybchenko <arybchenko@solarflare.com>
2018-10-17 10:26:59 +02:00

2736 lines
77 KiB
C
Raw Blame History

/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2014-2018 Chelsio Communications.
* All rights reserved.
*/
#include <sys/queue.h>
#include <stdio.h>
#include <errno.h>
#include <stdint.h>
#include <string.h>
#include <unistd.h>
#include <stdarg.h>
#include <inttypes.h>
#include <netinet/in.h>
#include <rte_byteorder.h>
#include <rte_common.h>
#include <rte_cycles.h>
#include <rte_interrupts.h>
#include <rte_log.h>
#include <rte_debug.h>
#include <rte_pci.h>
#include <rte_atomic.h>
#include <rte_branch_prediction.h>
#include <rte_memory.h>
#include <rte_memzone.h>
#include <rte_tailq.h>
#include <rte_eal.h>
#include <rte_alarm.h>
#include <rte_ether.h>
#include <rte_ethdev_driver.h>
#include <rte_malloc.h>
#include <rte_random.h>
#include <rte_dev.h>
#include "common.h"
#include "t4_regs.h"
#include "t4_msg.h"
#include "cxgbe.h"
static inline void ship_tx_pkt_coalesce_wr(struct adapter *adap,
struct sge_eth_txq *txq);
/*
* Max number of Rx buffers we replenish at a time.
*/
#define MAX_RX_REFILL 64U
#define NOMEM_TMR_IDX (SGE_NTIMERS - 1)
/*
* Max Tx descriptor space we allow for an Ethernet packet to be inlined
* into a WR.
*/
#define MAX_IMM_TX_PKT_LEN 256
/*
* Max size of a WR sent through a control Tx queue.
*/
#define MAX_CTRL_WR_LEN SGE_MAX_WR_LEN
/*
* Rx buffer sizes for "usembufs" Free List buffers (one ingress packet
* per mbuf buffer). We currently only support two sizes for 1500- and
* 9000-byte MTUs. We could easily support more but there doesn't seem to be
* much need for that ...
*/
#define FL_MTU_SMALL 1500
#define FL_MTU_LARGE 9000
static inline unsigned int fl_mtu_bufsize(struct adapter *adapter,
unsigned int mtu)
{
struct sge *s = &adapter->sge;
return CXGBE_ALIGN(s->pktshift + ETHER_HDR_LEN + VLAN_HLEN + mtu,
s->fl_align);
}
#define FL_MTU_SMALL_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_SMALL)
#define FL_MTU_LARGE_BUFSIZE(adapter) fl_mtu_bufsize(adapter, FL_MTU_LARGE)
/*
* Bits 0..3 of rx_sw_desc.dma_addr have special meaning. The hardware uses
* these to specify the buffer size as an index into the SGE Free List Buffer
* Size register array. We also use bit 4, when the buffer has been unmapped
* for DMA, but this is of course never sent to the hardware and is only used
* to prevent double unmappings. All of the above requires that the Free List
* Buffers which we allocate have the bottom 5 bits free (0) -- i.e. are
* 32-byte or or a power of 2 greater in alignment. Since the SGE's minimal
* Free List Buffer alignment is 32 bytes, this works out for us ...
*/
enum {
RX_BUF_FLAGS = 0x1f, /* bottom five bits are special */
RX_BUF_SIZE = 0x0f, /* bottom three bits are for buf sizes */
RX_UNMAPPED_BUF = 0x10, /* buffer is not mapped */
/*
* XXX We shouldn't depend on being able to use these indices.
* XXX Especially when some other Master PF has initialized the
* XXX adapter or we use the Firmware Configuration File. We
* XXX should really search through the Host Buffer Size register
* XXX array for the appropriately sized buffer indices.
*/
RX_SMALL_PG_BUF = 0x0, /* small (PAGE_SIZE) page buffer */
RX_LARGE_PG_BUF = 0x1, /* buffer large page buffer */
RX_SMALL_MTU_BUF = 0x2, /* small MTU buffer */
RX_LARGE_MTU_BUF = 0x3, /* large MTU buffer */
};
/**
* txq_avail - return the number of available slots in a Tx queue
* @q: the Tx queue
*
* Returns the number of descriptors in a Tx queue available to write new
* packets.
*/
static inline unsigned int txq_avail(const struct sge_txq *q)
{
return q->size - 1 - q->in_use;
}
static int map_mbuf(struct rte_mbuf *mbuf, dma_addr_t *addr)
{
struct rte_mbuf *m = mbuf;
for (; m; m = m->next, addr++) {
*addr = m->buf_iova + rte_pktmbuf_headroom(m);
if (*addr == 0)
goto out_err;
}
return 0;
out_err:
return -ENOMEM;
}
/**
* free_tx_desc - reclaims Tx descriptors and their buffers
* @q: the Tx queue to reclaim descriptors from
* @n: the number of descriptors to reclaim
*
* Reclaims Tx descriptors from an SGE Tx queue and frees the associated
* Tx buffers. Called with the Tx queue lock held.
*/
static void free_tx_desc(struct sge_txq *q, unsigned int n)
{
struct tx_sw_desc *d;
unsigned int cidx = 0;
d = &q->sdesc[cidx];
while (n--) {
if (d->mbuf) { /* an SGL is present */
rte_pktmbuf_free(d->mbuf);
d->mbuf = NULL;
}
if (d->coalesce.idx) {
int i;
for (i = 0; i < d->coalesce.idx; i++) {
rte_pktmbuf_free(d->coalesce.mbuf[i]);
d->coalesce.mbuf[i] = NULL;
}
d->coalesce.idx = 0;
}
++d;
if (++cidx == q->size) {
cidx = 0;
d = q->sdesc;
}
RTE_MBUF_PREFETCH_TO_FREE(&q->sdesc->mbuf->pool);
}
}
static void reclaim_tx_desc(struct sge_txq *q, unsigned int n)
{
struct tx_sw_desc *d;
unsigned int cidx = q->cidx;
d = &q->sdesc[cidx];
while (n--) {
if (d->mbuf) { /* an SGL is present */
rte_pktmbuf_free(d->mbuf);
d->mbuf = NULL;
}
++d;
if (++cidx == q->size) {
cidx = 0;
d = q->sdesc;
}
}
q->cidx = cidx;
}
/**
* fl_cap - return the capacity of a free-buffer list
* @fl: the FL
*
* Returns the capacity of a free-buffer list. The capacity is less than
* the size because one descriptor needs to be left unpopulated, otherwise
* HW will think the FL is empty.
*/
static inline unsigned int fl_cap(const struct sge_fl *fl)
{
return fl->size - 8; /* 1 descriptor = 8 buffers */
}
/**
* fl_starving - return whether a Free List is starving.
* @adapter: pointer to the adapter
* @fl: the Free List
*
* Tests specified Free List to see whether the number of buffers
* available to the hardware has falled below our "starvation"
* threshold.
*/
static inline bool fl_starving(const struct adapter *adapter,
const struct sge_fl *fl)
{
const struct sge *s = &adapter->sge;
return fl->avail - fl->pend_cred <= s->fl_starve_thres;
}
static inline unsigned int get_buf_size(struct adapter *adapter,
const struct rx_sw_desc *d)
{
unsigned int rx_buf_size_idx = d->dma_addr & RX_BUF_SIZE;
unsigned int buf_size = 0;
switch (rx_buf_size_idx) {
case RX_SMALL_MTU_BUF:
buf_size = FL_MTU_SMALL_BUFSIZE(adapter);
break;
case RX_LARGE_MTU_BUF:
buf_size = FL_MTU_LARGE_BUFSIZE(adapter);
break;
default:
BUG_ON(1);
/* NOT REACHED */
}
return buf_size;
}
/**
* free_rx_bufs - free the Rx buffers on an SGE free list
* @q: the SGE free list to free buffers from
* @n: how many buffers to free
*
* Release the next @n buffers on an SGE free-buffer Rx queue. The
* buffers must be made inaccessible to HW before calling this function.
*/
static void free_rx_bufs(struct sge_fl *q, int n)
{
unsigned int cidx = q->cidx;
struct rx_sw_desc *d;
d = &q->sdesc[cidx];
while (n--) {
if (d->buf) {
rte_pktmbuf_free(d->buf);
d->buf = NULL;
}
++d;
if (++cidx == q->size) {
cidx = 0;
d = q->sdesc;
}
q->avail--;
}
q->cidx = cidx;
}
/**
* unmap_rx_buf - unmap the current Rx buffer on an SGE free list
* @q: the SGE free list
*
* Unmap the current buffer on an SGE free-buffer Rx queue. The
* buffer must be made inaccessible to HW before calling this function.
*
* This is similar to @free_rx_bufs above but does not free the buffer.
* Do note that the FL still loses any further access to the buffer.
*/
static void unmap_rx_buf(struct sge_fl *q)
{
if (++q->cidx == q->size)
q->cidx = 0;
q->avail--;
}
static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q)
{
if (q->pend_cred >= 64) {
u32 val = adap->params.arch.sge_fl_db;
if (is_t4(adap->params.chip))
val |= V_PIDX(q->pend_cred / 8);
else
val |= V_PIDX_T5(q->pend_cred / 8);
/*
* Make sure all memory writes to the Free List queue are
* committed before we tell the hardware about them.
*/
wmb();
/*
* If we don't have access to the new User Doorbell (T5+), use
* the old doorbell mechanism; otherwise use the new BAR2
* mechanism.
*/
if (unlikely(!q->bar2_addr)) {
u32 reg = is_pf4(adap) ? MYPF_REG(A_SGE_PF_KDOORBELL) :
T4VF_SGE_BASE_ADDR +
A_SGE_VF_KDOORBELL;
t4_write_reg_relaxed(adap, reg,
val | V_QID(q->cntxt_id));
} else {
writel_relaxed(val | V_QID(q->bar2_qid),
(void *)((uintptr_t)q->bar2_addr +
SGE_UDB_KDOORBELL));
/*
* This Write memory Barrier will force the write to
* the User Doorbell area to be flushed.
*/
wmb();
}
q->pend_cred &= 7;
}
}
static inline void set_rx_sw_desc(struct rx_sw_desc *sd, void *buf,
dma_addr_t mapping)
{
sd->buf = buf;
sd->dma_addr = mapping; /* includes size low bits */
}
/**
* refill_fl_usembufs - refill an SGE Rx buffer ring with mbufs
* @adap: the adapter
* @q: the ring to refill
* @n: the number of new buffers to allocate
*
* (Re)populate an SGE free-buffer queue with up to @n new packet buffers,
* allocated with the supplied gfp flags. The caller must assure that
* @n does not exceed the queue's capacity. If afterwards the queue is
* found critically low mark it as starving in the bitmap of starving FLs.
*
* Returns the number of buffers allocated.
*/
static unsigned int refill_fl_usembufs(struct adapter *adap, struct sge_fl *q,
int n)
{
struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, fl);
unsigned int cred = q->avail;
__be64 *d = &q->desc[q->pidx];
struct rx_sw_desc *sd = &q->sdesc[q->pidx];
unsigned int buf_size_idx = RX_SMALL_MTU_BUF;
struct rte_mbuf *buf_bulk[n];
int ret, i;
struct rte_pktmbuf_pool_private *mbp_priv;
u8 jumbo_en = rxq->rspq.eth_dev->data->dev_conf.rxmode.offloads &
DEV_RX_OFFLOAD_JUMBO_FRAME;
/* Use jumbo mtu buffers if mbuf data room size can fit jumbo data. */
mbp_priv = rte_mempool_get_priv(rxq->rspq.mb_pool);
if (jumbo_en &&
((mbp_priv->mbuf_data_room_size - RTE_PKTMBUF_HEADROOM) >= 9000))
buf_size_idx = RX_LARGE_MTU_BUF;
ret = rte_mempool_get_bulk(rxq->rspq.mb_pool, (void *)buf_bulk, n);
if (unlikely(ret != 0)) {
dev_debug(adap, "%s: failed to allocated fl entries in bulk ..\n",
__func__);
q->alloc_failed++;
rxq->rspq.eth_dev->data->rx_mbuf_alloc_failed++;
goto out;
}
for (i = 0; i < n; i++) {
struct rte_mbuf *mbuf = buf_bulk[i];
dma_addr_t mapping;
if (!mbuf) {
dev_debug(adap, "%s: mbuf alloc failed\n", __func__);
q->alloc_failed++;
rxq->rspq.eth_dev->data->rx_mbuf_alloc_failed++;
goto out;
}
rte_mbuf_refcnt_set(mbuf, 1);
mbuf->data_off =
(uint16_t)(RTE_PTR_ALIGN((char *)mbuf->buf_addr +
RTE_PKTMBUF_HEADROOM,
adap->sge.fl_align) -
(char *)mbuf->buf_addr);
mbuf->next = NULL;
mbuf->nb_segs = 1;
mbuf->port = rxq->rspq.port_id;
mapping = (dma_addr_t)RTE_ALIGN(mbuf->buf_iova +
mbuf->data_off,
adap->sge.fl_align);
mapping |= buf_size_idx;
*d++ = cpu_to_be64(mapping);
set_rx_sw_desc(sd, mbuf, mapping);
sd++;
q->avail++;
if (++q->pidx == q->size) {
q->pidx = 0;
sd = q->sdesc;
d = q->desc;
}
}
out: cred = q->avail - cred;
q->pend_cred += cred;
ring_fl_db(adap, q);
if (unlikely(fl_starving(adap, q))) {
/*
* Make sure data has been written to free list
*/
wmb();
q->low++;
}
return cred;
}
/**
* refill_fl - refill an SGE Rx buffer ring with mbufs
* @adap: the adapter
* @q: the ring to refill
* @n: the number of new buffers to allocate
*
* (Re)populate an SGE free-buffer queue with up to @n new packet buffers,
* allocated with the supplied gfp flags. The caller must assure that
* @n does not exceed the queue's capacity. Returns the number of buffers
* allocated.
*/
static unsigned int refill_fl(struct adapter *adap, struct sge_fl *q, int n)
{
return refill_fl_usembufs(adap, q, n);
}
static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
{
refill_fl(adap, fl, min(MAX_RX_REFILL, fl_cap(fl) - fl->avail));
}
/*
* Return the number of reclaimable descriptors in a Tx queue.
*/
static inline int reclaimable(const struct sge_txq *q)
{
int hw_cidx = ntohs(q->stat->cidx);
hw_cidx -= q->cidx;
if (hw_cidx < 0)
return hw_cidx + q->size;
return hw_cidx;
}
/**
* reclaim_completed_tx - reclaims completed Tx descriptors
* @q: the Tx queue to reclaim completed descriptors from
*
* Reclaims Tx descriptors that the SGE has indicated it has processed.
*/
void reclaim_completed_tx(struct sge_txq *q)
{
unsigned int avail = reclaimable(q);
do {
/* reclaim as much as possible */
reclaim_tx_desc(q, avail);
q->in_use -= avail;
avail = reclaimable(q);
} while (avail);
}
/**
* sgl_len - calculates the size of an SGL of the given capacity
* @n: the number of SGL entries
*
* Calculates the number of flits needed for a scatter/gather list that
* can hold the given number of entries.
*/
static inline unsigned int sgl_len(unsigned int n)
{
/*
* A Direct Scatter Gather List uses 32-bit lengths and 64-bit PCI DMA
* addresses. The DSGL Work Request starts off with a 32-bit DSGL
* ULPTX header, then Length0, then Address0, then, for 1 <= i <= N,
* repeated sequences of { Length[i], Length[i+1], Address[i],
* Address[i+1] } (this ensures that all addresses are on 64-bit
* boundaries). If N is even, then Length[N+1] should be set to 0 and
* Address[N+1] is omitted.
*
* The following calculation incorporates all of the above. It's
* somewhat hard to follow but, briefly: the "+2" accounts for the
* first two flits which include the DSGL header, Length0 and
* Address0; the "(3*(n-1))/2" covers the main body of list entries (3
* flits for every pair of the remaining N) +1 if (n-1) is odd; and
* finally the "+((n-1)&1)" adds the one remaining flit needed if
* (n-1) is odd ...
*/
n--;
return (3 * n) / 2 + (n & 1) + 2;
}
/**
* flits_to_desc - returns the num of Tx descriptors for the given flits
* @n: the number of flits
*
* Returns the number of Tx descriptors needed for the supplied number
* of flits.
*/
static inline unsigned int flits_to_desc(unsigned int n)
{
return DIV_ROUND_UP(n, 8);
}
/**
* is_eth_imm - can an Ethernet packet be sent as immediate data?
* @m: the packet
*
* Returns whether an Ethernet packet is small enough to fit as
* immediate data. Return value corresponds to the headroom required.
*/
static inline int is_eth_imm(const struct rte_mbuf *m)
{
unsigned int hdrlen = (m->ol_flags & PKT_TX_TCP_SEG) ?
sizeof(struct cpl_tx_pkt_lso_core) : 0;
hdrlen += sizeof(struct cpl_tx_pkt);
if (m->pkt_len <= MAX_IMM_TX_PKT_LEN - hdrlen)
return hdrlen;
return 0;
}
/**
* calc_tx_flits - calculate the number of flits for a packet Tx WR
* @m: the packet
* @adap: adapter structure pointer
*
* Returns the number of flits needed for a Tx WR for the given Ethernet
* packet, including the needed WR and CPL headers.
*/
static inline unsigned int calc_tx_flits(const struct rte_mbuf *m,
struct adapter *adap)
{
size_t wr_size = is_pf4(adap) ? sizeof(struct fw_eth_tx_pkt_wr) :
sizeof(struct fw_eth_tx_pkt_vm_wr);
unsigned int flits;
int hdrlen;
/*
* If the mbuf is small enough, we can pump it out as a work request
* with only immediate data. In that case we just have to have the
* TX Packet header plus the mbuf data in the Work Request.
*/
hdrlen = is_eth_imm(m);
if (hdrlen)
return DIV_ROUND_UP(m->pkt_len + hdrlen, sizeof(__be64));
/*
* Otherwise, we're going to have to construct a Scatter gather list
* of the mbuf body and fragments. We also include the flits necessary
* for the TX Packet Work Request and CPL. We always have a firmware
* Write Header (incorporated as part of the cpl_tx_pkt_lso and
* cpl_tx_pkt structures), followed by either a TX Packet Write CPL
* message or, if we're doing a Large Send Offload, an LSO CPL message
* with an embedded TX Packet Write CPL message.
*/
flits = sgl_len(m->nb_segs);
if (m->tso_segsz)
flits += (wr_size + sizeof(struct cpl_tx_pkt_lso_core) +
sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
else
flits += (wr_size +
sizeof(struct cpl_tx_pkt_core)) / sizeof(__be64);
return flits;
}
/**
* write_sgl - populate a scatter/gather list for a packet
* @mbuf: the packet
* @q: the Tx queue we are writing into
* @sgl: starting location for writing the SGL
* @end: points right after the end of the SGL
* @start: start offset into mbuf main-body data to include in the SGL
* @addr: address of mapped region
*
* Generates a scatter/gather list for the buffers that make up a packet.
* The caller must provide adequate space for the SGL that will be written.
* The SGL includes all of the packet's page fragments and the data in its
* main body except for the first @start bytes. @sgl must be 16-byte
* aligned and within a Tx descriptor with available space. @end points
* write after the end of the SGL but does not account for any potential
* wrap around, i.e., @end > @sgl.
*/
static void write_sgl(struct rte_mbuf *mbuf, struct sge_txq *q,
struct ulptx_sgl *sgl, u64 *end, unsigned int start,
const dma_addr_t *addr)
{
unsigned int i, len;
struct ulptx_sge_pair *to;
struct rte_mbuf *m = mbuf;
unsigned int nfrags = m->nb_segs;
struct ulptx_sge_pair buf[nfrags / 2];
len = m->data_len - start;
sgl->len0 = htonl(len);
sgl->addr0 = rte_cpu_to_be_64(addr[0]);
sgl->cmd_nsge = htonl(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
V_ULPTX_NSGE(nfrags));
if (likely(--nfrags == 0))
return;
/*
* Most of the complexity below deals with the possibility we hit the
* end of the queue in the middle of writing the SGL. For this case
* only we create the SGL in a temporary buffer and then copy it.
*/
to = (u8 *)end > (u8 *)q->stat ? buf : sgl->sge;
for (i = 0; nfrags >= 2; nfrags -= 2, to++) {
m = m->next;
to->len[0] = rte_cpu_to_be_32(m->data_len);
to->addr[0] = rte_cpu_to_be_64(addr[++i]);
m = m->next;
to->len[1] = rte_cpu_to_be_32(m->data_len);
to->addr[1] = rte_cpu_to_be_64(addr[++i]);
}
if (nfrags) {
m = m->next;
to->len[0] = rte_cpu_to_be_32(m->data_len);
to->len[1] = rte_cpu_to_be_32(0);
to->addr[0] = rte_cpu_to_be_64(addr[i + 1]);
}
if (unlikely((u8 *)end > (u8 *)q->stat)) {
unsigned int part0 = RTE_PTR_DIFF((u8 *)q->stat,
(u8 *)sgl->sge);
unsigned int part1;
if (likely(part0))
memcpy(sgl->sge, buf, part0);
part1 = RTE_PTR_DIFF((u8 *)end, (u8 *)q->stat);
rte_memcpy(q->desc, RTE_PTR_ADD((u8 *)buf, part0), part1);
end = RTE_PTR_ADD((void *)q->desc, part1);
}
if ((uintptr_t)end & 8) /* 0-pad to multiple of 16 */
*(u64 *)end = 0;
}
#define IDXDIFF(head, tail, wrap) \
((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))
#define Q_IDXDIFF(q, idx) IDXDIFF((q)->pidx, (q)->idx, (q)->size)
#define R_IDXDIFF(q, idx) IDXDIFF((q)->cidx, (q)->idx, (q)->size)
#define PIDXDIFF(head, tail, wrap) \
((tail) >= (head) ? (tail) - (head) : (wrap) - (head) + (tail))
#define P_IDXDIFF(q, idx) PIDXDIFF((q)->cidx, idx, (q)->size)
/**
* ring_tx_db - ring a Tx queue's doorbell
* @adap: the adapter
* @q: the Tx queue
* @n: number of new descriptors to give to HW
*
* Ring the doorbel for a Tx queue.
*/
static inline void ring_tx_db(struct adapter *adap, struct sge_txq *q)
{
int n = Q_IDXDIFF(q, dbidx);
/*
* Make sure that all writes to the TX Descriptors are committed
* before we tell the hardware about them.
*/
rte_wmb();
/*
* If we don't have access to the new User Doorbell (T5+), use the old
* doorbell mechanism; otherwise use the new BAR2 mechanism.
*/
if (unlikely(!q->bar2_addr)) {
u32 val = V_PIDX(n);
/*
* For T4 we need to participate in the Doorbell Recovery
* mechanism.
*/
if (!q->db_disabled)
t4_write_reg(adap, MYPF_REG(A_SGE_PF_KDOORBELL),
V_QID(q->cntxt_id) | val);
else
q->db_pidx_inc += n;
q->db_pidx = q->pidx;
} else {
u32 val = V_PIDX_T5(n);
/*
* T4 and later chips share the same PIDX field offset within
* the doorbell, but T5 and later shrank the field in order to
* gain a bit for Doorbell Priority. The field was absurdly
* large in the first place (14 bits) so we just use the T5
* and later limits and warn if a Queue ID is too large.
*/
WARN_ON(val & F_DBPRIO);
writel(val | V_QID(q->bar2_qid),
(void *)((uintptr_t)q->bar2_addr + SGE_UDB_KDOORBELL));
/*
* This Write Memory Barrier will force the write to the User
* Doorbell area to be flushed. This is needed to prevent
* writes on different CPUs for the same queue from hitting
* the adapter out of order. This is required when some Work
* Requests take the Write Combine Gather Buffer path (user
* doorbell area offset [SGE_UDB_WCDOORBELL..+63]) and some
* take the traditional path where we simply increment the
* PIDX (User Doorbell area SGE_UDB_KDOORBELL) and have the
* hardware DMA read the actual Work Request.
*/
rte_wmb();
}
q->dbidx = q->pidx;
}
/*
* Figure out what HW csum a packet wants and return the appropriate control
* bits.
*/
static u64 hwcsum(enum chip_type chip, const struct rte_mbuf *m)
{
int csum_type;
if (m->ol_flags & PKT_TX_IP_CKSUM) {
switch (m->ol_flags & PKT_TX_L4_MASK) {
case PKT_TX_TCP_CKSUM:
csum_type = TX_CSUM_TCPIP;
break;
case PKT_TX_UDP_CKSUM:
csum_type = TX_CSUM_UDPIP;
break;
default:
goto nocsum;
}
} else {
goto nocsum;
}
if (likely(csum_type >= TX_CSUM_TCPIP)) {
u64 hdr_len = V_TXPKT_IPHDR_LEN(m->l3_len);
int eth_hdr_len = m->l2_len;
if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5)
hdr_len |= V_TXPKT_ETHHDR_LEN(eth_hdr_len);
else
hdr_len |= V_T6_TXPKT_ETHHDR_LEN(eth_hdr_len);
return V_TXPKT_CSUM_TYPE(csum_type) | hdr_len;
}
nocsum:
/*
* unknown protocol, disable HW csum
* and hope a bad packet is detected
*/
return F_TXPKT_L4CSUM_DIS;
}
static inline void txq_advance(struct sge_txq *q, unsigned int n)
{
q->in_use += n;
q->pidx += n;
if (q->pidx >= q->size)
q->pidx -= q->size;
}
#define MAX_COALESCE_LEN 64000
static inline int wraps_around(struct sge_txq *q, int ndesc)
{
return (q->pidx + ndesc) > q->size ? 1 : 0;
}
static void tx_timer_cb(void *data)
{
struct adapter *adap = (struct adapter *)data;
struct sge_eth_txq *txq = &adap->sge.ethtxq[0];
int i;
unsigned int coal_idx;
/* monitor any pending tx */
for (i = 0; i < adap->sge.max_ethqsets; i++, txq++) {
if (t4_os_trylock(&txq->txq_lock)) {
coal_idx = txq->q.coalesce.idx;
if (coal_idx) {
if (coal_idx == txq->q.last_coal_idx &&
txq->q.pidx == txq->q.last_pidx) {
ship_tx_pkt_coalesce_wr(adap, txq);
} else {
txq->q.last_coal_idx = coal_idx;
txq->q.last_pidx = txq->q.pidx;
}
}
t4_os_unlock(&txq->txq_lock);
}
}
rte_eal_alarm_set(50, tx_timer_cb, (void *)adap);
}
/**
* ship_tx_pkt_coalesce_wr - finalizes and ships a coalesce WR
* @ adap: adapter structure
* @txq: tx queue
*
* writes the different fields of the pkts WR and sends it.
*/
static inline void ship_tx_pkt_coalesce_wr(struct adapter *adap,
struct sge_eth_txq *txq)
{
struct fw_eth_tx_pkts_vm_wr *vmwr;
const size_t fw_hdr_copy_len = (sizeof(vmwr->ethmacdst) +
sizeof(vmwr->ethmacsrc) +
sizeof(vmwr->ethtype) +
sizeof(vmwr->vlantci));
struct fw_eth_tx_pkts_wr *wr;
struct sge_txq *q = &txq->q;
unsigned int ndesc;
u32 wr_mid;
/* fill the pkts WR header */
wr = (void *)&q->desc[q->pidx];
wr->op_pkd = htonl(V_FW_WR_OP(FW_ETH_TX_PKTS2_WR));
vmwr = (void *)&q->desc[q->pidx];
wr_mid = V_FW_WR_LEN16(DIV_ROUND_UP(q->coalesce.flits, 2));
ndesc = flits_to_desc(q->coalesce.flits);
wr->equiq_to_len16 = htonl(wr_mid);
wr->plen = cpu_to_be16(q->coalesce.len);
wr->npkt = q->coalesce.idx;
wr->r3 = 0;
if (is_pf4(adap)) {
wr->op_pkd = htonl(V_FW_WR_OP(FW_ETH_TX_PKTS2_WR));
wr->type = q->coalesce.type;
} else {
wr->op_pkd = htonl(V_FW_WR_OP(FW_ETH_TX_PKTS_VM_WR));
vmwr->r4 = 0;
memcpy((void *)vmwr->ethmacdst, (void *)q->coalesce.ethmacdst,
fw_hdr_copy_len);
}
/* zero out coalesce structure members */
memset((void *)&q->coalesce, 0, sizeof(struct eth_coalesce));
txq_advance(q, ndesc);
txq->stats.coal_wr++;
txq->stats.coal_pkts += wr->npkt;
if (Q_IDXDIFF(q, equeidx) >= q->size / 2) {
q->equeidx = q->pidx;
wr_mid |= F_FW_WR_EQUEQ;
wr->equiq_to_len16 = htonl(wr_mid);
}
ring_tx_db(adap, q);
}
/**
* should_tx_packet_coalesce - decides wether to coalesce an mbuf or not
* @txq: tx queue where the mbuf is sent
* @mbuf: mbuf to be sent
* @nflits: return value for number of flits needed
* @adap: adapter structure
*
* This function decides if a packet should be coalesced or not.
*/
static inline int should_tx_packet_coalesce(struct sge_eth_txq *txq,
struct rte_mbuf *mbuf,
unsigned int *nflits,
struct adapter *adap)
{
struct fw_eth_tx_pkts_vm_wr *wr;
const size_t fw_hdr_copy_len = (sizeof(wr->ethmacdst) +
sizeof(wr->ethmacsrc) +
sizeof(wr->ethtype) +
sizeof(wr->vlantci));
struct sge_txq *q = &txq->q;
unsigned int flits, ndesc;
unsigned char type = 0;
int credits, wr_size;
/* use coal WR type 1 when no frags are present */
type = (mbuf->nb_segs == 1) ? 1 : 0;
if (!is_pf4(adap)) {
if (!type)
return 0;
if (q->coalesce.idx && memcmp((void *)q->coalesce.ethmacdst,
rte_pktmbuf_mtod(mbuf, void *),
fw_hdr_copy_len))
ship_tx_pkt_coalesce_wr(adap, txq);
}
if (unlikely(type != q->coalesce.type && q->coalesce.idx))
ship_tx_pkt_coalesce_wr(adap, txq);
/* calculate the number of flits required for coalescing this packet
* without the 2 flits of the WR header. These are added further down
* if we are just starting in new PKTS WR. sgl_len doesn't account for
* the possible 16 bytes alignment ULP TX commands so we do it here.
*/
flits = (sgl_len(mbuf->nb_segs) + 1) & ~1U;
if (type == 0)
flits += (sizeof(struct ulp_txpkt) +
sizeof(struct ulptx_idata)) / sizeof(__be64);
flits += sizeof(struct cpl_tx_pkt_core) / sizeof(__be64);
*nflits = flits;
/* If coalescing is on, the mbuf is added to a pkts WR */
if (q->coalesce.idx) {
ndesc = DIV_ROUND_UP(q->coalesce.flits + flits, 8);
credits = txq_avail(q) - ndesc;
/* If we are wrapping or this is last mbuf then, send the
* already coalesced mbufs and let the non-coalesce pass
* handle the mbuf.
*/
if (unlikely(credits < 0 || wraps_around(q, ndesc))) {
ship_tx_pkt_coalesce_wr(adap, txq);
return 0;
}
/* If the max coalesce len or the max WR len is reached
* ship the WR and keep coalescing on.
*/
if (unlikely((q->coalesce.len + mbuf->pkt_len >
MAX_COALESCE_LEN) ||
(q->coalesce.flits + flits >
q->coalesce.max))) {
ship_tx_pkt_coalesce_wr(adap, txq);
goto new;
}
return 1;
}
new:
/* start a new pkts WR, the WR header is not filled below */
wr_size = is_pf4(adap) ? sizeof(struct fw_eth_tx_pkts_wr) :
sizeof(struct fw_eth_tx_pkts_vm_wr);
flits += wr_size / sizeof(__be64);
ndesc = flits_to_desc(q->coalesce.flits + flits);
credits = txq_avail(q) - ndesc;
if (unlikely(credits < 0 || wraps_around(q, ndesc)))
return 0;
q->coalesce.flits += wr_size / sizeof(__be64);
q->coalesce.type = type;
q->coalesce.ptr = (unsigned char *)&q->desc[q->pidx] +
q->coalesce.flits * sizeof(__be64);
if (!is_pf4(adap))
memcpy((void *)q->coalesce.ethmacdst,
rte_pktmbuf_mtod(mbuf, void *), fw_hdr_copy_len);
return 1;
}
/**
* tx_do_packet_coalesce - add an mbuf to a coalesce WR
* @txq: sge_eth_txq used send the mbuf
* @mbuf: mbuf to be sent
* @flits: flits needed for this mbuf
* @adap: adapter structure
* @pi: port_info structure
* @addr: mapped address of the mbuf
*
* Adds an mbuf to be sent as part of a coalesce WR by filling a
* ulp_tx_pkt command, ulp_tx_sc_imm command, cpl message and
* ulp_tx_sc_dsgl command.
*/
static inline int tx_do_packet_coalesce(struct sge_eth_txq *txq,
struct rte_mbuf *mbuf,
int flits, struct adapter *adap,
const struct port_info *pi,
dma_addr_t *addr, uint16_t nb_pkts)
{
u64 cntrl, *end;
struct sge_txq *q = &txq->q;
struct ulp_txpkt *mc;
struct ulptx_idata *sc_imm;
struct cpl_tx_pkt_core *cpl;
struct tx_sw_desc *sd;
unsigned int idx = q->coalesce.idx, len = mbuf->pkt_len;
unsigned int max_coal_pkt_num = is_pf4(adap) ? ETH_COALESCE_PKT_NUM :
ETH_COALESCE_VF_PKT_NUM;
#ifdef RTE_LIBRTE_CXGBE_TPUT
RTE_SET_USED(nb_pkts);
#endif
if (q->coalesce.type == 0) {
mc = (struct ulp_txpkt *)q->coalesce.ptr;
mc->cmd_dest = htonl(V_ULPTX_CMD(4) | V_ULP_TXPKT_DEST(0) |
V_ULP_TXPKT_FID(adap->sge.fw_evtq.cntxt_id) |
F_ULP_TXPKT_RO);
mc->len = htonl(DIV_ROUND_UP(flits, 2));
sc_imm = (struct ulptx_idata *)(mc + 1);
sc_imm->cmd_more = htonl(V_ULPTX_CMD(ULP_TX_SC_IMM) |
F_ULP_TX_SC_MORE);
sc_imm->len = htonl(sizeof(*cpl));
end = (u64 *)mc + flits;
cpl = (struct cpl_tx_pkt_core *)(sc_imm + 1);
} else {
end = (u64 *)q->coalesce.ptr + flits;
cpl = (struct cpl_tx_pkt_core *)q->coalesce.ptr;
}
/* update coalesce structure for this txq */
q->coalesce.flits += flits;
q->coalesce.ptr += flits * sizeof(__be64);
q->coalesce.len += mbuf->pkt_len;
/* fill the cpl message, same as in t4_eth_xmit, this should be kept
* similar to t4_eth_xmit
*/
if (mbuf->ol_flags & PKT_TX_IP_CKSUM) {
cntrl = hwcsum(adap->params.chip, mbuf) |
F_TXPKT_IPCSUM_DIS;
txq->stats.tx_cso++;
} else {
cntrl = F_TXPKT_L4CSUM_DIS | F_TXPKT_IPCSUM_DIS;
}
if (mbuf->ol_flags & PKT_TX_VLAN_PKT) {
txq->stats.vlan_ins++;
cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(mbuf->vlan_tci);
}
cpl->ctrl0 = htonl(V_TXPKT_OPCODE(CPL_TX_PKT_XT));
if (is_pf4(adap))
cpl->ctrl0 |= htonl(V_TXPKT_INTF(pi->tx_chan) |
V_TXPKT_PF(adap->pf));
else
cpl->ctrl0 |= htonl(V_TXPKT_INTF(pi->port_id));
cpl->pack = htons(0);
cpl->len = htons(len);
cpl->ctrl1 = cpu_to_be64(cntrl);
write_sgl(mbuf, q, (struct ulptx_sgl *)(cpl + 1), end, 0, addr);
txq->stats.pkts++;
txq->stats.tx_bytes += len;
sd = &q->sdesc[q->pidx + (idx >> 1)];
if (!(idx & 1)) {
if (sd->coalesce.idx) {
int i;
for (i = 0; i < sd->coalesce.idx; i++) {
rte_pktmbuf_free(sd->coalesce.mbuf[i]);
sd->coalesce.mbuf[i] = NULL;
}
}
}
/* store pointers to the mbuf and the sgl used in free_tx_desc.
* each tx desc can hold two pointers corresponding to the value
* of ETH_COALESCE_PKT_PER_DESC
*/
sd->coalesce.mbuf[idx & 1] = mbuf;
sd->coalesce.sgl[idx & 1] = (struct ulptx_sgl *)(cpl + 1);
sd->coalesce.idx = (idx & 1) + 1;
/* send the coaelsced work request if max reached */
if (++q->coalesce.idx == max_coal_pkt_num
#ifndef RTE_LIBRTE_CXGBE_TPUT
|| q->coalesce.idx >= nb_pkts
#endif
)
ship_tx_pkt_coalesce_wr(adap, txq);
return 0;
}
/**
* t4_eth_xmit - add a packet to an Ethernet Tx queue
* @txq: the egress queue
* @mbuf: the packet
*
* Add a packet to an SGE Ethernet Tx queue. Runs with softirqs disabled.
*/
int t4_eth_xmit(struct sge_eth_txq *txq, struct rte_mbuf *mbuf,
uint16_t nb_pkts)
{
const struct port_info *pi;
struct cpl_tx_pkt_lso_core *lso;
struct adapter *adap;
struct rte_mbuf *m = mbuf;
struct fw_eth_tx_pkt_wr *wr;
struct fw_eth_tx_pkt_vm_wr *vmwr;
struct cpl_tx_pkt_core *cpl;
struct tx_sw_desc *d;
dma_addr_t addr[m->nb_segs];
unsigned int flits, ndesc, cflits;
int l3hdr_len, l4hdr_len, eth_xtra_len;
int len, last_desc;
int credits;
u32 wr_mid;
u64 cntrl, *end;
bool v6;
u32 max_pkt_len = txq->data->dev_conf.rxmode.max_rx_pkt_len;
/* Reject xmit if queue is stopped */
if (unlikely(txq->flags & EQ_STOPPED))
return -(EBUSY);
/*
* The chip min packet length is 10 octets but play safe and reject
* anything shorter than an Ethernet header.
*/
if (unlikely(m->pkt_len < ETHER_HDR_LEN)) {
out_free:
rte_pktmbuf_free(m);
return 0;
}
if ((!(m->ol_flags & PKT_TX_TCP_SEG)) &&
(unlikely(m->pkt_len > max_pkt_len)))
goto out_free;
pi = (struct port_info *)txq->data->dev_private;
adap = pi->adapter;
cntrl = F_TXPKT_L4CSUM_DIS | F_TXPKT_IPCSUM_DIS;
/* align the end of coalesce WR to a 512 byte boundary */
txq->q.coalesce.max = (8 - (txq->q.pidx & 7)) * 8;
if (!((m->ol_flags & PKT_TX_TCP_SEG) || (m->pkt_len > ETHER_MAX_LEN))) {
if (should_tx_packet_coalesce(txq, mbuf, &cflits, adap)) {
if (unlikely(map_mbuf(mbuf, addr) < 0)) {
dev_warn(adap, "%s: mapping err for coalesce\n",
__func__);
txq->stats.mapping_err++;
goto out_free;
}
rte_prefetch0((volatile void *)addr);
return tx_do_packet_coalesce(txq, mbuf, cflits, adap,
pi, addr, nb_pkts);
} else {
return -EBUSY;
}
}
if (txq->q.coalesce.idx)
ship_tx_pkt_coalesce_wr(adap, txq);
flits = calc_tx_flits(m, adap);
ndesc = flits_to_desc(flits);
credits = txq_avail(&txq->q) - ndesc;
if (unlikely(credits < 0)) {
dev_debug(adap, "%s: Tx ring %u full; credits = %d\n",
__func__, txq->q.cntxt_id, credits);
return -EBUSY;
}
if (unlikely(map_mbuf(m, addr) < 0)) {
txq->stats.mapping_err++;
goto out_free;
}
wr_mid = V_FW_WR_LEN16(DIV_ROUND_UP(flits, 2));
if (Q_IDXDIFF(&txq->q, equeidx) >= 64) {
txq->q.equeidx = txq->q.pidx;
wr_mid |= F_FW_WR_EQUEQ;
}
wr = (void *)&txq->q.desc[txq->q.pidx];
vmwr = (void *)&txq->q.desc[txq->q.pidx];
wr->equiq_to_len16 = htonl(wr_mid);
if (is_pf4(adap)) {
wr->r3 = rte_cpu_to_be_64(0);
end = (u64 *)wr + flits;
} else {
const size_t fw_hdr_copy_len = (sizeof(vmwr->ethmacdst) +
sizeof(vmwr->ethmacsrc) +
sizeof(vmwr->ethtype) +
sizeof(vmwr->vlantci));
vmwr->r3[0] = rte_cpu_to_be_32(0);
vmwr->r3[1] = rte_cpu_to_be_32(0);
memcpy((void *)vmwr->ethmacdst, rte_pktmbuf_mtod(m, void *),
fw_hdr_copy_len);
end = (u64 *)vmwr + flits;
}
len = 0;
len += sizeof(*cpl);
/* Coalescing skipped and we send through normal path */
if (!(m->ol_flags & PKT_TX_TCP_SEG)) {
wr->op_immdlen = htonl(V_FW_WR_OP(is_pf4(adap) ?
FW_ETH_TX_PKT_WR :
FW_ETH_TX_PKT_VM_WR) |
V_FW_WR_IMMDLEN(len));
if (is_pf4(adap))
cpl = (void *)(wr + 1);
else
cpl = (void *)(vmwr + 1);
if (m->ol_flags & PKT_TX_IP_CKSUM) {
cntrl = hwcsum(adap->params.chip, m) |
F_TXPKT_IPCSUM_DIS;
txq->stats.tx_cso++;
}
} else {
if (is_pf4(adap))
lso = (void *)(wr + 1);
else
lso = (void *)(vmwr + 1);
v6 = (m->ol_flags & PKT_TX_IPV6) != 0;
l3hdr_len = m->l3_len;
l4hdr_len = m->l4_len;
eth_xtra_len = m->l2_len - ETHER_HDR_LEN;
len += sizeof(*lso);
wr->op_immdlen = htonl(V_FW_WR_OP(is_pf4(adap) ?
FW_ETH_TX_PKT_WR :
FW_ETH_TX_PKT_VM_WR) |
V_FW_WR_IMMDLEN(len));
lso->lso_ctrl = htonl(V_LSO_OPCODE(CPL_TX_PKT_LSO) |
F_LSO_FIRST_SLICE | F_LSO_LAST_SLICE |
V_LSO_IPV6(v6) |
V_LSO_ETHHDR_LEN(eth_xtra_len / 4) |
V_LSO_IPHDR_LEN(l3hdr_len / 4) |
V_LSO_TCPHDR_LEN(l4hdr_len / 4));
lso->ipid_ofst = htons(0);
lso->mss = htons(m->tso_segsz);
lso->seqno_offset = htonl(0);
if (is_t4(adap->params.chip))
lso->len = htonl(m->pkt_len);
else
lso->len = htonl(V_LSO_T5_XFER_SIZE(m->pkt_len));
cpl = (void *)(lso + 1);
if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5)
cntrl = V_TXPKT_ETHHDR_LEN(eth_xtra_len);
else
cntrl = V_T6_TXPKT_ETHHDR_LEN(eth_xtra_len);
cntrl |= V_TXPKT_CSUM_TYPE(v6 ? TX_CSUM_TCPIP6 :
TX_CSUM_TCPIP) |
V_TXPKT_IPHDR_LEN(l3hdr_len);
txq->stats.tso++;
txq->stats.tx_cso += m->tso_segsz;
}
if (m->ol_flags & PKT_TX_VLAN_PKT) {
txq->stats.vlan_ins++;
cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(m->vlan_tci);
}
cpl->ctrl0 = htonl(V_TXPKT_OPCODE(CPL_TX_PKT_XT));
if (is_pf4(adap))
cpl->ctrl0 |= htonl(V_TXPKT_INTF(pi->tx_chan) |
V_TXPKT_PF(adap->pf));
else
cpl->ctrl0 |= htonl(V_TXPKT_INTF(pi->port_id) |
V_TXPKT_PF(0));
cpl->pack = htons(0);
cpl->len = htons(m->pkt_len);
cpl->ctrl1 = cpu_to_be64(cntrl);
txq->stats.pkts++;
txq->stats.tx_bytes += m->pkt_len;
last_desc = txq->q.pidx + ndesc - 1;
if (last_desc >= (int)txq->q.size)
last_desc -= txq->q.size;
d = &txq->q.sdesc[last_desc];
if (d->coalesce.idx) {
int i;
for (i = 0; i < d->coalesce.idx; i++) {
rte_pktmbuf_free(d->coalesce.mbuf[i]);
d->coalesce.mbuf[i] = NULL;
}
d->coalesce.idx = 0;
}
write_sgl(m, &txq->q, (struct ulptx_sgl *)(cpl + 1), end, 0,
addr);
txq->q.sdesc[last_desc].mbuf = m;
txq->q.sdesc[last_desc].sgl = (struct ulptx_sgl *)(cpl + 1);
txq_advance(&txq->q, ndesc);
ring_tx_db(adap, &txq->q);
return 0;
}
/**
* reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
* @q: the SGE control Tx queue
*
* This is a variant of reclaim_completed_tx() that is used for Tx queues
* that send only immediate data (presently just the control queues) and
* thus do not have any mbufs to release.
*/
static inline void reclaim_completed_tx_imm(struct sge_txq *q)
{
int hw_cidx = ntohs(q->stat->cidx);
int reclaim = hw_cidx - q->cidx;
if (reclaim < 0)
reclaim += q->size;
q->in_use -= reclaim;
q->cidx = hw_cidx;
}
/**
* is_imm - check whether a packet can be sent as immediate data
* @mbuf: the packet
*
* Returns true if a packet can be sent as a WR with immediate data.
*/
static inline int is_imm(const struct rte_mbuf *mbuf)
{
return mbuf->pkt_len <= MAX_CTRL_WR_LEN;
}
/**
* inline_tx_mbuf: inline a packet's data into TX descriptors
* @q: the TX queue where the packet will be inlined
* @from: pointer to data portion of packet
* @to: pointer after cpl where data has to be inlined
* @len: length of data to inline
*
* Inline a packet's contents directly to TX descriptors, starting at
* the given position within the TX DMA ring.
* Most of the complexity of this operation is dealing with wrap arounds
* in the middle of the packet we want to inline.
*/
static void inline_tx_mbuf(const struct sge_txq *q, caddr_t from, caddr_t *to,
int len)
{
int left = RTE_PTR_DIFF(q->stat, *to);
if (likely((uintptr_t)*to + len <= (uintptr_t)q->stat)) {
rte_memcpy(*to, from, len);
*to = RTE_PTR_ADD(*to, len);
} else {
rte_memcpy(*to, from, left);
from = RTE_PTR_ADD(from, left);
left = len - left;
rte_memcpy((void *)q->desc, from, left);
*to = RTE_PTR_ADD((void *)q->desc, left);
}
}
/**
* ctrl_xmit - send a packet through an SGE control Tx queue
* @q: the control queue
* @mbuf: the packet
*
* Send a packet through an SGE control Tx queue. Packets sent through
* a control queue must fit entirely as immediate data.
*/
static int ctrl_xmit(struct sge_ctrl_txq *q, struct rte_mbuf *mbuf)
{
unsigned int ndesc;
struct fw_wr_hdr *wr;
caddr_t dst;
if (unlikely(!is_imm(mbuf))) {
WARN_ON(1);
rte_pktmbuf_free(mbuf);
return -1;
}
reclaim_completed_tx_imm(&q->q);
ndesc = DIV_ROUND_UP(mbuf->pkt_len, sizeof(struct tx_desc));
t4_os_lock(&q->ctrlq_lock);
q->full = txq_avail(&q->q) < ndesc ? 1 : 0;
if (unlikely(q->full)) {
t4_os_unlock(&q->ctrlq_lock);
return -1;
}
wr = (struct fw_wr_hdr *)&q->q.desc[q->q.pidx];
dst = (void *)wr;
inline_tx_mbuf(&q->q, rte_pktmbuf_mtod(mbuf, caddr_t),
&dst, mbuf->data_len);
txq_advance(&q->q, ndesc);
if (unlikely(txq_avail(&q->q) < 64))
wr->lo |= htonl(F_FW_WR_EQUEQ);
q->txp++;
ring_tx_db(q->adapter, &q->q);
t4_os_unlock(&q->ctrlq_lock);
rte_pktmbuf_free(mbuf);
return 0;
}
/**
* t4_mgmt_tx - send a management message
* @q: the control queue
* @mbuf: the packet containing the management message
*
* Send a management message through control queue.
*/
int t4_mgmt_tx(struct sge_ctrl_txq *q, struct rte_mbuf *mbuf)
{
return ctrl_xmit(q, mbuf);
}
/**
* alloc_ring - allocate resources for an SGE descriptor ring
* @dev: the PCI device's core device
* @nelem: the number of descriptors
* @elem_size: the size of each descriptor
* @sw_size: the size of the SW state associated with each ring element
* @phys: the physical address of the allocated ring
* @metadata: address of the array holding the SW state for the ring
* @stat_size: extra space in HW ring for status information
* @node: preferred node for memory allocations
*
* Allocates resources for an SGE descriptor ring, such as Tx queues,
* free buffer lists, or response queues. Each SGE ring requires
* space for its HW descriptors plus, optionally, space for the SW state
* associated with each HW entry (the metadata). The function returns
* three values: the virtual address for the HW ring (the return value
* of the function), the bus address of the HW ring, and the address
* of the SW ring.
*/
static void *alloc_ring(size_t nelem, size_t elem_size,
size_t sw_size, dma_addr_t *phys, void *metadata,
size_t stat_size, __rte_unused uint16_t queue_id,
int socket_id, const char *z_name,
const char *z_name_sw)
{
size_t len = CXGBE_MAX_RING_DESC_SIZE * elem_size + stat_size;
const struct rte_memzone *tz;
void *s = NULL;
dev_debug(adapter, "%s: nelem = %zu; elem_size = %zu; sw_size = %zu; "
"stat_size = %zu; queue_id = %u; socket_id = %d; z_name = %s;"
" z_name_sw = %s\n", __func__, nelem, elem_size, sw_size,
stat_size, queue_id, socket_id, z_name, z_name_sw);
tz = rte_memzone_lookup(z_name);
if (tz) {
dev_debug(adapter, "%s: tz exists...returning existing..\n",
__func__);
goto alloc_sw_ring;
}
/*
* Allocate TX/RX ring hardware descriptors. A memzone large enough to
* handle the maximum ring size is allocated in order to allow for
* resizing in later calls to the queue setup function.
*/
tz = rte_memzone_reserve_aligned(z_name, len, socket_id,
RTE_MEMZONE_IOVA_CONTIG, 4096);
if (!tz)
return NULL;
alloc_sw_ring:
memset(tz->addr, 0, len);
if (sw_size) {
s = rte_zmalloc_socket(z_name_sw, nelem * sw_size,
RTE_CACHE_LINE_SIZE, socket_id);
if (!s) {
dev_err(adapter, "%s: failed to get sw_ring memory\n",
__func__);
return NULL;
}
}
if (metadata)
*(void **)metadata = s;
*phys = (uint64_t)tz->iova;
return tz->addr;
}
/**
* t4_pktgl_to_mbuf_usembufs - build an mbuf from a packet gather list
* @gl: the gather list
*
* Builds an mbuf from the given packet gather list. Returns the mbuf or
* %NULL if mbuf allocation failed.
*/
static struct rte_mbuf *t4_pktgl_to_mbuf_usembufs(const struct pkt_gl *gl)
{
/*
* If there's only one mbuf fragment, just return that.
*/
if (likely(gl->nfrags == 1))
return gl->mbufs[0];
return NULL;
}
/**
* t4_pktgl_to_mbuf - build an mbuf from a packet gather list
* @gl: the gather list
*
* Builds an mbuf from the given packet gather list. Returns the mbuf or
* %NULL if mbuf allocation failed.
*/
static struct rte_mbuf *t4_pktgl_to_mbuf(const struct pkt_gl *gl)
{
return t4_pktgl_to_mbuf_usembufs(gl);
}
/**
* t4_ethrx_handler - process an ingress ethernet packet
* @q: the response queue that received the packet
* @rsp: the response queue descriptor holding the RX_PKT message
* @si: the gather list of packet fragments
*
* Process an ingress ethernet packet and deliver it to the stack.
*/
int t4_ethrx_handler(struct sge_rspq *q, const __be64 *rsp,
const struct pkt_gl *si)
{
struct rte_mbuf *mbuf;
const struct cpl_rx_pkt *pkt;
const struct rss_header *rss_hdr;
bool csum_ok;
struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq);
u16 err_vec;
rss_hdr = (const void *)rsp;
pkt = (const void *)&rsp[1];
/* Compressed error vector is enabled for T6 only */
if (q->adapter->params.tp.rx_pkt_encap)
err_vec = G_T6_COMPR_RXERR_VEC(ntohs(pkt->err_vec));
else
err_vec = ntohs(pkt->err_vec);
csum_ok = pkt->csum_calc && !err_vec;
mbuf = t4_pktgl_to_mbuf(si);
if (unlikely(!mbuf)) {
rxq->stats.rx_drops++;
return 0;
}
mbuf->port = pkt->iff;
if (pkt->l2info & htonl(F_RXF_IP)) {
mbuf->packet_type = RTE_PTYPE_L3_IPV4;
if (unlikely(!csum_ok))
mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
if ((pkt->l2info & htonl(F_RXF_UDP | F_RXF_TCP)) && !csum_ok)
mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
} else if (pkt->l2info & htonl(F_RXF_IP6)) {
mbuf->packet_type = RTE_PTYPE_L3_IPV6;
}
mbuf->port = pkt->iff;
if (!rss_hdr->filter_tid && rss_hdr->hash_type) {
mbuf->ol_flags |= PKT_RX_RSS_HASH;
mbuf->hash.rss = ntohl(rss_hdr->hash_val);
}
if (pkt->vlan_ex) {
mbuf->ol_flags |= PKT_RX_VLAN;
mbuf->vlan_tci = ntohs(pkt->vlan);
}
rxq->stats.pkts++;
rxq->stats.rx_bytes += mbuf->pkt_len;
return 0;
}
#define CXGB4_MSG_AN ((void *)1)
/**
* rspq_next - advance to the next entry in a response queue
* @q: the queue
*
* Updates the state of a response queue to advance it to the next entry.
*/
static inline void rspq_next(struct sge_rspq *q)
{
q->cur_desc = (const __be64 *)((const char *)q->cur_desc + q->iqe_len);
if (unlikely(++q->cidx == q->size)) {
q->cidx = 0;
q->gen ^= 1;
q->cur_desc = q->desc;
}
}
/**
* process_responses - process responses from an SGE response queue
* @q: the ingress queue to process
* @budget: how many responses can be processed in this round
* @rx_pkts: mbuf to put the pkts
*
* Process responses from an SGE response queue up to the supplied budget.
* Responses include received packets as well as control messages from FW
* or HW.
*
* Additionally choose the interrupt holdoff time for the next interrupt
* on this queue. If the system is under memory shortage use a fairly
* long delay to help recovery.
*/
static int process_responses(struct sge_rspq *q, int budget,
struct rte_mbuf **rx_pkts)
{
int ret = 0, rsp_type;
int budget_left = budget;
const struct rsp_ctrl *rc;
struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq);
while (likely(budget_left)) {
if (q->cidx == ntohs(q->stat->pidx))
break;
rc = (const struct rsp_ctrl *)
((const char *)q->cur_desc + (q->iqe_len - sizeof(*rc)));
/*
* Ensure response has been read
*/
rmb();
rsp_type = G_RSPD_TYPE(rc->u.type_gen);
if (likely(rsp_type == X_RSPD_TYPE_FLBUF)) {
struct sge *s = &q->adapter->sge;
unsigned int stat_pidx;
int stat_pidx_diff;
stat_pidx = ntohs(q->stat->pidx);
stat_pidx_diff = P_IDXDIFF(q, stat_pidx);
while (stat_pidx_diff && budget_left) {
const struct rx_sw_desc *rsd =
&rxq->fl.sdesc[rxq->fl.cidx];
const struct rss_header *rss_hdr =
(const void *)q->cur_desc;
const struct cpl_rx_pkt *cpl =
(const void *)&q->cur_desc[1];
struct rte_mbuf *pkt, *npkt;
u32 len, bufsz;
bool csum_ok;
u16 err_vec;
rc = (const struct rsp_ctrl *)
((const char *)q->cur_desc +
(q->iqe_len - sizeof(*rc)));
rsp_type = G_RSPD_TYPE(rc->u.type_gen);
if (unlikely(rsp_type != X_RSPD_TYPE_FLBUF))
break;
len = ntohl(rc->pldbuflen_qid);
BUG_ON(!(len & F_RSPD_NEWBUF));
pkt = rsd->buf;
npkt = pkt;
len = G_RSPD_LEN(len);
pkt->pkt_len = len;
/* Compressed error vector is enabled for
* T6 only
*/
if (q->adapter->params.tp.rx_pkt_encap)
err_vec = G_T6_COMPR_RXERR_VEC(
ntohs(cpl->err_vec));
else
err_vec = ntohs(cpl->err_vec);
csum_ok = cpl->csum_calc && !err_vec;
/* Chain mbufs into len if necessary */
while (len) {
struct rte_mbuf *new_pkt = rsd->buf;
bufsz = min(get_buf_size(q->adapter,
rsd), len);
new_pkt->data_len = bufsz;
unmap_rx_buf(&rxq->fl);
len -= bufsz;
npkt->next = new_pkt;
npkt = new_pkt;
pkt->nb_segs++;
rsd = &rxq->fl.sdesc[rxq->fl.cidx];
}
npkt->next = NULL;
pkt->nb_segs--;
if (cpl->l2info & htonl(F_RXF_IP)) {
pkt->packet_type = RTE_PTYPE_L3_IPV4;
if (unlikely(!csum_ok))
pkt->ol_flags |=
PKT_RX_IP_CKSUM_BAD;
if ((cpl->l2info &
htonl(F_RXF_UDP | F_RXF_TCP)) &&
!csum_ok)
pkt->ol_flags |=
PKT_RX_L4_CKSUM_BAD;
} else if (cpl->l2info & htonl(F_RXF_IP6)) {
pkt->packet_type = RTE_PTYPE_L3_IPV6;
}
if (!rss_hdr->filter_tid &&
rss_hdr->hash_type) {
pkt->ol_flags |= PKT_RX_RSS_HASH;
pkt->hash.rss =
ntohl(rss_hdr->hash_val);
}
if (cpl->vlan_ex) {
pkt->ol_flags |= PKT_RX_VLAN |
PKT_RX_VLAN_STRIPPED;
pkt->vlan_tci = ntohs(cpl->vlan);
}
rte_pktmbuf_adj(pkt, s->pktshift);
rxq->stats.pkts++;
rxq->stats.rx_bytes += pkt->pkt_len;
rx_pkts[budget - budget_left] = pkt;
rspq_next(q);
budget_left--;
stat_pidx_diff--;
}
continue;
} else if (likely(rsp_type == X_RSPD_TYPE_CPL)) {
ret = q->handler(q, q->cur_desc, NULL);
} else {
ret = q->handler(q, (const __be64 *)rc, CXGB4_MSG_AN);
}
if (unlikely(ret)) {
/* couldn't process descriptor, back off for recovery */
q->next_intr_params = V_QINTR_TIMER_IDX(NOMEM_TMR_IDX);
break;
}
rspq_next(q);
budget_left--;
}
/*
* If this is a Response Queue with an associated Free List and
* there's room for another chunk of new Free List buffer pointers,
* refill the Free List.
*/
if (q->offset >= 0 && fl_cap(&rxq->fl) - rxq->fl.avail >= 64)
__refill_fl(q->adapter, &rxq->fl);
return budget - budget_left;
}
int cxgbe_poll(struct sge_rspq *q, struct rte_mbuf **rx_pkts,
unsigned int budget, unsigned int *work_done)
{
struct sge_eth_rxq *rxq = container_of(q, struct sge_eth_rxq, rspq);
unsigned int cidx_inc;
unsigned int params;
u32 val;
*work_done = process_responses(q, budget, rx_pkts);
if (*work_done) {
cidx_inc = R_IDXDIFF(q, gts_idx);
if (q->offset >= 0 && fl_cap(&rxq->fl) - rxq->fl.avail >= 64)
__refill_fl(q->adapter, &rxq->fl);
params = q->intr_params;
q->next_intr_params = params;
val = V_CIDXINC(cidx_inc) | V_SEINTARM(params);
if (unlikely(!q->bar2_addr)) {
u32 reg = is_pf4(q->adapter) ? MYPF_REG(A_SGE_PF_GTS) :
T4VF_SGE_BASE_ADDR +
A_SGE_VF_GTS;
t4_write_reg(q->adapter, reg,
val | V_INGRESSQID((u32)q->cntxt_id));
} else {
writel(val | V_INGRESSQID(q->bar2_qid),
(void *)((uintptr_t)q->bar2_addr + SGE_UDB_GTS));
/* This Write memory Barrier will force the
* write to the User Doorbell area to be
* flushed.
*/
wmb();
}
q->gts_idx = q->cidx;
}
return 0;
}
/**
* bar2_address - return the BAR2 address for an SGE Queue's Registers
* @adapter: the adapter
* @qid: the SGE Queue ID
* @qtype: the SGE Queue Type (Egress or Ingress)
* @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues
*
* Returns the BAR2 address for the SGE Queue Registers associated with
* @qid. If BAR2 SGE Registers aren't available, returns NULL. Also
* returns the BAR2 Queue ID to be used with writes to the BAR2 SGE
* Queue Registers. If the BAR2 Queue ID is 0, then "Inferred Queue ID"
* Registers are supported (e.g. the Write Combining Doorbell Buffer).
*/
static void __iomem *bar2_address(struct adapter *adapter, unsigned int qid,
enum t4_bar2_qtype qtype,
unsigned int *pbar2_qid)
{
u64 bar2_qoffset;
int ret;
ret = t4_bar2_sge_qregs(adapter, qid, qtype, &bar2_qoffset, pbar2_qid);
if (ret)
return NULL;
return adapter->bar2 + bar2_qoffset;
}
int t4_sge_eth_rxq_start(struct adapter *adap, struct sge_rspq *rq)
{
struct sge_eth_rxq *rxq = container_of(rq, struct sge_eth_rxq, rspq);
unsigned int fl_id = rxq->fl.size ? rxq->fl.cntxt_id : 0xffff;
return t4_iq_start_stop(adap, adap->mbox, true, adap->pf, 0,
rq->cntxt_id, fl_id, 0xffff);
}
int t4_sge_eth_rxq_stop(struct adapter *adap, struct sge_rspq *rq)
{
struct sge_eth_rxq *rxq = container_of(rq, struct sge_eth_rxq, rspq);
unsigned int fl_id = rxq->fl.size ? rxq->fl.cntxt_id : 0xffff;
return t4_iq_start_stop(adap, adap->mbox, false, adap->pf, 0,
rq->cntxt_id, fl_id, 0xffff);
}
/*
* @intr_idx: MSI/MSI-X vector if >=0, -(absolute qid + 1) if < 0
* @cong: < 0 -> no congestion feedback, >= 0 -> congestion channel map
*/
int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
struct rte_eth_dev *eth_dev, int intr_idx,
struct sge_fl *fl, rspq_handler_t hnd, int cong,
struct rte_mempool *mp, int queue_id, int socket_id)
{
int ret, flsz = 0;
struct fw_iq_cmd c;
struct sge *s = &adap->sge;
struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
char z_name[RTE_MEMZONE_NAMESIZE];
char z_name_sw[RTE_MEMZONE_NAMESIZE];
unsigned int nb_refill;
u8 pciechan;
/* Size needs to be multiple of 16, including status entry. */
iq->size = cxgbe_roundup(iq->size, 16);
snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
eth_dev->data->port_id, queue_id,
fwevtq ? "fwq_ring" : "rx_ring");
snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
iq->desc = alloc_ring(iq->size, iq->iqe_len, 0, &iq->phys_addr, NULL, 0,
queue_id, socket_id, z_name, z_name_sw);
if (!iq->desc)
return -ENOMEM;
memset(&c, 0, sizeof(c));
c.op_to_vfn = htonl(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
F_FW_CMD_WRITE | F_FW_CMD_EXEC);
if (is_pf4(adap)) {
pciechan = pi->tx_chan;
c.op_to_vfn |= htonl(V_FW_IQ_CMD_PFN(adap->pf) |
V_FW_IQ_CMD_VFN(0));
if (cong >= 0)
c.iqns_to_fl0congen =
htonl(F_FW_IQ_CMD_IQFLINTCONGEN |
V_FW_IQ_CMD_IQTYPE(cong ?
FW_IQ_IQTYPE_NIC :
FW_IQ_IQTYPE_OFLD) |
F_FW_IQ_CMD_IQRO);
} else {
pciechan = pi->port_id;
}
c.alloc_to_len16 = htonl(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
(sizeof(c) / 16));
c.type_to_iqandstindex =
htonl(V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
V_FW_IQ_CMD_IQASYNCH(fwevtq) |
V_FW_IQ_CMD_VIID(pi->viid) |
V_FW_IQ_CMD_IQANDST(intr_idx < 0) |
V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_STATUS_PAGE) |
V_FW_IQ_CMD_IQANDSTINDEX(intr_idx >= 0 ? intr_idx :
-intr_idx - 1));
c.iqdroprss_to_iqesize =
htons(V_FW_IQ_CMD_IQPCIECH(pciechan) |
F_FW_IQ_CMD_IQGTSMODE |
V_FW_IQ_CMD_IQINTCNTTHRESH(iq->pktcnt_idx) |
V_FW_IQ_CMD_IQESIZE(ilog2(iq->iqe_len) - 4));
c.iqsize = htons(iq->size);
c.iqaddr = cpu_to_be64(iq->phys_addr);
if (fl) {
struct sge_eth_rxq *rxq = container_of(fl, struct sge_eth_rxq,
fl);
unsigned int chip_ver = CHELSIO_CHIP_VERSION(adap->params.chip);
/*
* Allocate the ring for the hardware free list (with space
* for its status page) along with the associated software
* descriptor ring. The free list size needs to be a multiple
* of the Egress Queue Unit and at least 2 Egress Units larger
* than the SGE's Egress Congrestion Threshold
* (fl_starve_thres - 1).
*/
if (fl->size < s->fl_starve_thres - 1 + 2 * 8)
fl->size = s->fl_starve_thres - 1 + 2 * 8;
fl->size = cxgbe_roundup(fl->size, 8);
snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
eth_dev->data->port_id, queue_id,
fwevtq ? "fwq_ring" : "fl_ring");
snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
fl->desc = alloc_ring(fl->size, sizeof(__be64),
sizeof(struct rx_sw_desc),
&fl->addr, &fl->sdesc, s->stat_len,
queue_id, socket_id, z_name, z_name_sw);
if (!fl->desc)
goto fl_nomem;
flsz = fl->size / 8 + s->stat_len / sizeof(struct tx_desc);
c.iqns_to_fl0congen |=
htonl(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
(unlikely(rxq->usembufs) ?
0 : F_FW_IQ_CMD_FL0PACKEN) |
F_FW_IQ_CMD_FL0FETCHRO | F_FW_IQ_CMD_FL0DATARO |
F_FW_IQ_CMD_FL0PADEN);
if (is_pf4(adap) && cong >= 0)
c.iqns_to_fl0congen |=
htonl(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
F_FW_IQ_CMD_FL0CONGCIF |
F_FW_IQ_CMD_FL0CONGEN);
/* In T6, for egress queue type FL there is internal overhead
* of 16B for header going into FLM module.
* Hence maximum allowed burst size will be 448 bytes.
*/
c.fl0dcaen_to_fl0cidxfthresh =
htons(V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5 ?
X_FETCHBURSTMIN_128B :
X_FETCHBURSTMIN_64B) |
V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5 ?
X_FETCHBURSTMAX_512B :
X_FETCHBURSTMAX_256B));
c.fl0size = htons(flsz);
c.fl0addr = cpu_to_be64(fl->addr);
}
if (is_pf4(adap))
ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
else
ret = t4vf_wr_mbox(adap, &c, sizeof(c), &c);
if (ret)
goto err;
iq->cur_desc = iq->desc;
iq->cidx = 0;
iq->gts_idx = 0;
iq->gen = 1;
iq->next_intr_params = iq->intr_params;
iq->cntxt_id = ntohs(c.iqid);
iq->abs_id = ntohs(c.physiqid);
iq->bar2_addr = bar2_address(adap, iq->cntxt_id, T4_BAR2_QTYPE_INGRESS,
&iq->bar2_qid);
iq->size--; /* subtract status entry */
iq->stat = (void *)&iq->desc[iq->size * 8];
iq->eth_dev = eth_dev;
iq->handler = hnd;
iq->port_id = pi->pidx;
iq->mb_pool = mp;
/* set offset to -1 to distinguish ingress queues without FL */
iq->offset = fl ? 0 : -1;
if (fl) {
fl->cntxt_id = ntohs(c.fl0id);
fl->avail = 0;
fl->pend_cred = 0;
fl->pidx = 0;
fl->cidx = 0;
fl->alloc_failed = 0;
/*
* Note, we must initialize the BAR2 Free List User Doorbell
* information before refilling the Free List!
*/
fl->bar2_addr = bar2_address(adap, fl->cntxt_id,
T4_BAR2_QTYPE_EGRESS,
&fl->bar2_qid);
nb_refill = refill_fl(adap, fl, fl_cap(fl));
if (nb_refill != fl_cap(fl)) {
ret = -ENOMEM;
dev_err(adap, "%s: mbuf alloc failed with error: %d\n",
__func__, ret);
goto refill_fl_err;
}
}
/*
* For T5 and later we attempt to set up the Congestion Manager values
* of the new RX Ethernet Queue. This should really be handled by
* firmware because it's more complex than any host driver wants to
* get involved with and it's different per chip and this is almost
* certainly wrong. Formware would be wrong as well, but it would be
* a lot easier to fix in one place ... For now we do something very
* simple (and hopefully less wrong).
*/
if (is_pf4(adap) && !is_t4(adap->params.chip) && cong >= 0) {
u32 param, val;
int i;
param = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
V_FW_PARAMS_PARAM_YZ(iq->cntxt_id));
if (cong == 0) {
val = V_CONMCTXT_CNGTPMODE(X_CONMCTXT_CNGTPMODE_QUEUE);
} else {
val = V_CONMCTXT_CNGTPMODE(
X_CONMCTXT_CNGTPMODE_CHANNEL);
for (i = 0; i < 4; i++) {
if (cong & (1 << i))
val |= V_CONMCTXT_CNGCHMAP(1 <<
(i << 2));
}
}
ret = t4_set_params(adap, adap->mbox, adap->pf, 0, 1,
&param, &val);
if (ret)
dev_warn(adap->pdev_dev, "Failed to set Congestion Manager Context for Ingress Queue %d: %d\n",
iq->cntxt_id, -ret);
}
return 0;
refill_fl_err:
t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
iq->cntxt_id, fl->cntxt_id, 0xffff);
fl_nomem:
ret = -ENOMEM;
err:
iq->cntxt_id = 0;
iq->abs_id = 0;
if (iq->desc)
iq->desc = NULL;
if (fl && fl->desc) {
rte_free(fl->sdesc);
fl->cntxt_id = 0;
fl->sdesc = NULL;
fl->desc = NULL;
}
return ret;
}
static void init_txq(struct adapter *adap, struct sge_txq *q, unsigned int id,
unsigned int abs_id)
{
q->cntxt_id = id;
q->abs_id = abs_id;
q->bar2_addr = bar2_address(adap, q->cntxt_id, T4_BAR2_QTYPE_EGRESS,
&q->bar2_qid);
q->cidx = 0;
q->pidx = 0;
q->dbidx = 0;
q->in_use = 0;
q->equeidx = 0;
q->coalesce.idx = 0;
q->coalesce.len = 0;
q->coalesce.flits = 0;
q->last_coal_idx = 0;
q->last_pidx = 0;
q->stat = (void *)&q->desc[q->size];
}
int t4_sge_eth_txq_start(struct sge_eth_txq *txq)
{
/*
* TODO: For flow-control, queue may be stopped waiting to reclaim
* credits.
* Ensure queue is in EQ_STOPPED state before starting it.
*/
if (!(txq->flags & EQ_STOPPED))
return -(EBUSY);
txq->flags &= ~EQ_STOPPED;
return 0;
}
int t4_sge_eth_txq_stop(struct sge_eth_txq *txq)
{
txq->flags |= EQ_STOPPED;
return 0;
}
int t4_sge_alloc_eth_txq(struct adapter *adap, struct sge_eth_txq *txq,
struct rte_eth_dev *eth_dev, uint16_t queue_id,
unsigned int iqid, int socket_id)
{
int ret, nentries;
struct fw_eq_eth_cmd c;
struct sge *s = &adap->sge;
struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
char z_name[RTE_MEMZONE_NAMESIZE];
char z_name_sw[RTE_MEMZONE_NAMESIZE];
u8 pciechan;
/* Add status entries */
nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
eth_dev->data->port_id, queue_id, "tx_ring");
snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
txq->q.desc = alloc_ring(txq->q.size, sizeof(struct tx_desc),
sizeof(struct tx_sw_desc), &txq->q.phys_addr,
&txq->q.sdesc, s->stat_len, queue_id,
socket_id, z_name, z_name_sw);
if (!txq->q.desc)
return -ENOMEM;
memset(&c, 0, sizeof(c));
c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
F_FW_CMD_WRITE | F_FW_CMD_EXEC);
if (is_pf4(adap)) {
pciechan = pi->tx_chan;
c.op_to_vfn |= htonl(V_FW_EQ_ETH_CMD_PFN(adap->pf) |
V_FW_EQ_ETH_CMD_VFN(0));
} else {
pciechan = pi->port_id;
}
c.alloc_to_len16 = htonl(F_FW_EQ_ETH_CMD_ALLOC |
F_FW_EQ_ETH_CMD_EQSTART | (sizeof(c) / 16));
c.autoequiqe_to_viid = htonl(F_FW_EQ_ETH_CMD_AUTOEQUEQE |
V_FW_EQ_ETH_CMD_VIID(pi->viid));
c.fetchszm_to_iqid =
htonl(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
V_FW_EQ_ETH_CMD_PCIECHN(pciechan) |
F_FW_EQ_ETH_CMD_FETCHRO | V_FW_EQ_ETH_CMD_IQID(iqid));
c.dcaen_to_eqsize =
htonl(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
V_FW_EQ_ETH_CMD_EQSIZE(nentries));
c.eqaddr = rte_cpu_to_be_64(txq->q.phys_addr);
if (is_pf4(adap))
ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
else
ret = t4vf_wr_mbox(adap, &c, sizeof(c), &c);
if (ret) {
rte_free(txq->q.sdesc);
txq->q.sdesc = NULL;
txq->q.desc = NULL;
return ret;
}
init_txq(adap, &txq->q, G_FW_EQ_ETH_CMD_EQID(ntohl(c.eqid_pkd)),
G_FW_EQ_ETH_CMD_PHYSEQID(ntohl(c.physeqid_pkd)));
txq->stats.tso = 0;
txq->stats.pkts = 0;
txq->stats.tx_cso = 0;
txq->stats.coal_wr = 0;
txq->stats.vlan_ins = 0;
txq->stats.tx_bytes = 0;
txq->stats.coal_pkts = 0;
txq->stats.mapping_err = 0;
txq->flags |= EQ_STOPPED;
txq->eth_dev = eth_dev;
txq->data = eth_dev->data;
t4_os_lock_init(&txq->txq_lock);
return 0;
}
int t4_sge_alloc_ctrl_txq(struct adapter *adap, struct sge_ctrl_txq *txq,
struct rte_eth_dev *eth_dev, uint16_t queue_id,
unsigned int iqid, int socket_id)
{
int ret, nentries;
struct fw_eq_ctrl_cmd c;
struct sge *s = &adap->sge;
struct port_info *pi = (struct port_info *)(eth_dev->data->dev_private);
char z_name[RTE_MEMZONE_NAMESIZE];
char z_name_sw[RTE_MEMZONE_NAMESIZE];
/* Add status entries */
nentries = txq->q.size + s->stat_len / sizeof(struct tx_desc);
snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
eth_dev->data->port_id, queue_id, "ctrl_tx_ring");
snprintf(z_name_sw, sizeof(z_name_sw), "%s_sw_ring", z_name);
txq->q.desc = alloc_ring(txq->q.size, sizeof(struct tx_desc),
0, &txq->q.phys_addr,
NULL, 0, queue_id,
socket_id, z_name, z_name_sw);
if (!txq->q.desc)
return -ENOMEM;
memset(&c, 0, sizeof(c));
c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
F_FW_CMD_WRITE | F_FW_CMD_EXEC |
V_FW_EQ_CTRL_CMD_PFN(adap->pf) |
V_FW_EQ_CTRL_CMD_VFN(0));
c.alloc_to_len16 = htonl(F_FW_EQ_CTRL_CMD_ALLOC |
F_FW_EQ_CTRL_CMD_EQSTART | (sizeof(c) / 16));
c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(0));
c.physeqid_pkd = htonl(0);
c.fetchszm_to_iqid =
htonl(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_NONE) |
V_FW_EQ_CTRL_CMD_PCIECHN(pi->tx_chan) |
F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(iqid));
c.dcaen_to_eqsize =
htonl(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
V_FW_EQ_CTRL_CMD_EQSIZE(nentries));
c.eqaddr = cpu_to_be64(txq->q.phys_addr);
ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
if (ret) {
txq->q.desc = NULL;
return ret;
}
init_txq(adap, &txq->q, G_FW_EQ_CTRL_CMD_EQID(ntohl(c.cmpliqid_eqid)),
G_FW_EQ_CTRL_CMD_EQID(ntohl(c. physeqid_pkd)));
txq->adapter = adap;
txq->full = 0;
return 0;
}
static void free_txq(struct sge_txq *q)
{
q->cntxt_id = 0;
q->sdesc = NULL;
q->desc = NULL;
}
static void free_rspq_fl(struct adapter *adap, struct sge_rspq *rq,
struct sge_fl *fl)
{
unsigned int fl_id = fl ? fl->cntxt_id : 0xffff;
t4_iq_free(adap, adap->mbox, adap->pf, 0, FW_IQ_TYPE_FL_INT_CAP,
rq->cntxt_id, fl_id, 0xffff);
rq->cntxt_id = 0;
rq->abs_id = 0;
rq->desc = NULL;
if (fl) {
free_rx_bufs(fl, fl->avail);
rte_free(fl->sdesc);
fl->sdesc = NULL;
fl->cntxt_id = 0;
fl->desc = NULL;
}
}
/*
* Clear all queues of the port
*
* Note: This function must only be called after rx and tx path
* of the port have been disabled.
*/
void t4_sge_eth_clear_queues(struct port_info *pi)
{
int i;
struct adapter *adap = pi->adapter;
struct sge_eth_rxq *rxq = &adap->sge.ethrxq[pi->first_qset];
struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset];
for (i = 0; i < pi->n_rx_qsets; i++, rxq++) {
if (rxq->rspq.desc)
t4_sge_eth_rxq_stop(adap, &rxq->rspq);
}
for (i = 0; i < pi->n_tx_qsets; i++, txq++) {
if (txq->q.desc) {
struct sge_txq *q = &txq->q;
t4_sge_eth_txq_stop(txq);
reclaim_completed_tx(q);
free_tx_desc(q, q->size);
q->equeidx = q->pidx;
}
}
}
void t4_sge_eth_rxq_release(struct adapter *adap, struct sge_eth_rxq *rxq)
{
if (rxq->rspq.desc) {
t4_sge_eth_rxq_stop(adap, &rxq->rspq);
free_rspq_fl(adap, &rxq->rspq, rxq->fl.size ? &rxq->fl : NULL);
}
}
void t4_sge_eth_txq_release(struct adapter *adap, struct sge_eth_txq *txq)
{
if (txq->q.desc) {
t4_sge_eth_txq_stop(txq);
reclaim_completed_tx(&txq->q);
t4_eth_eq_free(adap, adap->mbox, adap->pf, 0, txq->q.cntxt_id);
free_tx_desc(&txq->q, txq->q.size);
rte_free(txq->q.sdesc);
free_txq(&txq->q);
}
}
void t4_sge_tx_monitor_start(struct adapter *adap)
{
rte_eal_alarm_set(50, tx_timer_cb, (void *)adap);
}
void t4_sge_tx_monitor_stop(struct adapter *adap)
{
rte_eal_alarm_cancel(tx_timer_cb, (void *)adap);
}
/**
* t4_free_sge_resources - free SGE resources
* @adap: the adapter
*
* Frees resources used by the SGE queue sets.
*/
void t4_free_sge_resources(struct adapter *adap)
{
unsigned int i;
struct sge_eth_rxq *rxq = &adap->sge.ethrxq[0];
struct sge_eth_txq *txq = &adap->sge.ethtxq[0];
/* clean up Ethernet Tx/Rx queues */
for (i = 0; i < adap->sge.max_ethqsets; i++, rxq++, txq++) {
/* Free only the queues allocated */
if (rxq->rspq.desc) {
t4_sge_eth_rxq_release(adap, rxq);
rxq->rspq.eth_dev = NULL;
}
if (txq->q.desc) {
t4_sge_eth_txq_release(adap, txq);
txq->eth_dev = NULL;
}
}
/* clean up control Tx queues */
for (i = 0; i < ARRAY_SIZE(adap->sge.ctrlq); i++) {
struct sge_ctrl_txq *cq = &adap->sge.ctrlq[i];
if (cq->q.desc) {
reclaim_completed_tx_imm(&cq->q);
t4_ctrl_eq_free(adap, adap->mbox, adap->pf, 0,
cq->q.cntxt_id);
free_txq(&cq->q);
}
}
if (adap->sge.fw_evtq.desc)
free_rspq_fl(adap, &adap->sge.fw_evtq, NULL);
}
/**
* t4_sge_init - initialize SGE
* @adap: the adapter
*
* Performs SGE initialization needed every time after a chip reset.
* We do not initialize any of the queues here, instead the driver
* top-level must request those individually.
*
* Called in two different modes:
*
* 1. Perform actual hardware initialization and record hard-coded
* parameters which were used. This gets used when we're the
* Master PF and the Firmware Configuration File support didn't
* work for some reason.
*
* 2. We're not the Master PF or initialization was performed with
* a Firmware Configuration File. In this case we need to grab
* any of the SGE operating parameters that we need to have in
* order to do our job and make sure we can live with them ...
*/
static int t4_sge_init_soft(struct adapter *adap)
{
struct sge *s = &adap->sge;
u32 fl_small_pg, fl_large_pg, fl_small_mtu, fl_large_mtu;
u32 timer_value_0_and_1, timer_value_2_and_3, timer_value_4_and_5;
u32 ingress_rx_threshold;
/*
* Verify that CPL messages are going to the Ingress Queue for
* process_responses() and that only packet data is going to the
* Free Lists.
*/
if ((t4_read_reg(adap, A_SGE_CONTROL) & F_RXPKTCPLMODE) !=
V_RXPKTCPLMODE(X_RXPKTCPLMODE_SPLIT)) {
dev_err(adap, "bad SGE CPL MODE\n");
return -EINVAL;
}
/*
* Validate the Host Buffer Register Array indices that we want to
* use ...
*
* XXX Note that we should really read through the Host Buffer Size
* XXX register array and find the indices of the Buffer Sizes which
* XXX meet our needs!
*/
#define READ_FL_BUF(x) \
t4_read_reg(adap, A_SGE_FL_BUFFER_SIZE0 + (x) * sizeof(u32))
fl_small_pg = READ_FL_BUF(RX_SMALL_PG_BUF);
fl_large_pg = READ_FL_BUF(RX_LARGE_PG_BUF);
fl_small_mtu = READ_FL_BUF(RX_SMALL_MTU_BUF);
fl_large_mtu = READ_FL_BUF(RX_LARGE_MTU_BUF);
/*
* We only bother using the Large Page logic if the Large Page Buffer
* is larger than our Page Size Buffer.
*/
if (fl_large_pg <= fl_small_pg)
fl_large_pg = 0;
#undef READ_FL_BUF
/*
* The Page Size Buffer must be exactly equal to our Page Size and the
* Large Page Size Buffer should be 0 (per above) or a power of 2.
*/
if (fl_small_pg != CXGBE_PAGE_SIZE ||
(fl_large_pg & (fl_large_pg - 1)) != 0) {
dev_err(adap, "bad SGE FL page buffer sizes [%d, %d]\n",
fl_small_pg, fl_large_pg);
return -EINVAL;
}
if (fl_large_pg)
s->fl_pg_order = ilog2(fl_large_pg) - PAGE_SHIFT;
if (adap->use_unpacked_mode) {
int err = 0;
if (fl_small_mtu < FL_MTU_SMALL_BUFSIZE(adap)) {
dev_err(adap, "bad SGE FL small MTU %d\n",
fl_small_mtu);
err = -EINVAL;
}
if (fl_large_mtu < FL_MTU_LARGE_BUFSIZE(adap)) {
dev_err(adap, "bad SGE FL large MTU %d\n",
fl_large_mtu);
err = -EINVAL;
}
if (err)
return err;
}
/*
* Retrieve our RX interrupt holdoff timer values and counter
* threshold values from the SGE parameters.
*/
timer_value_0_and_1 = t4_read_reg(adap, A_SGE_TIMER_VALUE_0_AND_1);
timer_value_2_and_3 = t4_read_reg(adap, A_SGE_TIMER_VALUE_2_AND_3);
timer_value_4_and_5 = t4_read_reg(adap, A_SGE_TIMER_VALUE_4_AND_5);
s->timer_val[0] = core_ticks_to_us(adap,
G_TIMERVALUE0(timer_value_0_and_1));
s->timer_val[1] = core_ticks_to_us(adap,
G_TIMERVALUE1(timer_value_0_and_1));
s->timer_val[2] = core_ticks_to_us(adap,
G_TIMERVALUE2(timer_value_2_and_3));
s->timer_val[3] = core_ticks_to_us(adap,
G_TIMERVALUE3(timer_value_2_and_3));
s->timer_val[4] = core_ticks_to_us(adap,
G_TIMERVALUE4(timer_value_4_and_5));
s->timer_val[5] = core_ticks_to_us(adap,
G_TIMERVALUE5(timer_value_4_and_5));
ingress_rx_threshold = t4_read_reg(adap, A_SGE_INGRESS_RX_THRESHOLD);
s->counter_val[0] = G_THRESHOLD_0(ingress_rx_threshold);
s->counter_val[1] = G_THRESHOLD_1(ingress_rx_threshold);
s->counter_val[2] = G_THRESHOLD_2(ingress_rx_threshold);
s->counter_val[3] = G_THRESHOLD_3(ingress_rx_threshold);
return 0;
}
int t4_sge_init(struct adapter *adap)
{
struct sge *s = &adap->sge;
u32 sge_control, sge_conm_ctrl;
int ret, egress_threshold;
/*
* Ingress Padding Boundary and Egress Status Page Size are set up by
* t4_fixup_host_params().
*/
sge_control = t4_read_reg(adap, A_SGE_CONTROL);
s->pktshift = G_PKTSHIFT(sge_control);
s->stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64;
s->fl_align = t4_fl_pkt_align(adap);
ret = t4_sge_init_soft(adap);
if (ret < 0) {
dev_err(adap, "%s: t4_sge_init_soft failed, error %d\n",
__func__, -ret);
return ret;
}
/*
* A FL with <= fl_starve_thres buffers is starving and a periodic
* timer will attempt to refill it. This needs to be larger than the
* SGE's Egress Congestion Threshold. If it isn't, then we can get
* stuck waiting for new packets while the SGE is waiting for us to
* give it more Free List entries. (Note that the SGE's Egress
* Congestion Threshold is in units of 2 Free List pointers.) For T4,
* there was only a single field to control this. For T5 there's the
* original field which now only applies to Unpacked Mode Free List
* buffers and a new field which only applies to Packed Mode Free List
* buffers.
*/
sge_conm_ctrl = t4_read_reg(adap, A_SGE_CONM_CTRL);
if (is_t4(adap->params.chip) || adap->use_unpacked_mode)
egress_threshold = G_EGRTHRESHOLD(sge_conm_ctrl);
else
egress_threshold = G_EGRTHRESHOLDPACKING(sge_conm_ctrl);
s->fl_starve_thres = 2 * egress_threshold + 1;
return 0;
}
int t4vf_sge_init(struct adapter *adap)
{
struct sge_params *sge_params = &adap->params.sge;
u32 sge_ingress_queues_per_page;
u32 sge_egress_queues_per_page;
u32 sge_control, sge_control2;
u32 fl_small_pg, fl_large_pg;
u32 sge_ingress_rx_threshold;
u32 sge_timer_value_0_and_1;
u32 sge_timer_value_2_and_3;
u32 sge_timer_value_4_and_5;
u32 sge_congestion_control;
struct sge *s = &adap->sge;
unsigned int s_hps, s_qpp;
u32 sge_host_page_size;
u32 params[7], vals[7];
int v;
/* query basic params from fw */
params[0] = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_REG) |
V_FW_PARAMS_PARAM_XYZ(A_SGE_CONTROL));
params[1] = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_REG) |
V_FW_PARAMS_PARAM_XYZ(A_SGE_HOST_PAGE_SIZE));
params[2] = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_REG) |
V_FW_PARAMS_PARAM_XYZ(A_SGE_FL_BUFFER_SIZE0));
params[3] = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_REG) |
V_FW_PARAMS_PARAM_XYZ(A_SGE_FL_BUFFER_SIZE1));
params[4] = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_REG) |
V_FW_PARAMS_PARAM_XYZ(A_SGE_TIMER_VALUE_0_AND_1));
params[5] = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_REG) |
V_FW_PARAMS_PARAM_XYZ(A_SGE_TIMER_VALUE_2_AND_3));
params[6] = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_REG) |
V_FW_PARAMS_PARAM_XYZ(A_SGE_TIMER_VALUE_4_AND_5));
v = t4vf_query_params(adap, 7, params, vals);
if (v != FW_SUCCESS)
return v;
sge_control = vals[0];
sge_host_page_size = vals[1];
fl_small_pg = vals[2];
fl_large_pg = vals[3];
sge_timer_value_0_and_1 = vals[4];
sge_timer_value_2_and_3 = vals[5];
sge_timer_value_4_and_5 = vals[6];
/*
* Start by vetting the basic SGE parameters which have been set up by
* the Physical Function Driver.
*/
/* We only bother using the Large Page logic if the Large Page Buffer
* is larger than our Page Size Buffer.
*/
if (fl_large_pg <= fl_small_pg)
fl_large_pg = 0;
/* The Page Size Buffer must be exactly equal to our Page Size and the
* Large Page Size Buffer should be 0 (per above) or a power of 2.
*/
if (fl_small_pg != CXGBE_PAGE_SIZE ||
(fl_large_pg & (fl_large_pg - 1)) != 0) {
dev_err(adapter->pdev_dev, "bad SGE FL buffer sizes [%d, %d]\n",
fl_small_pg, fl_large_pg);
return -EINVAL;
}
if ((sge_control & F_RXPKTCPLMODE) !=
V_RXPKTCPLMODE(X_RXPKTCPLMODE_SPLIT)) {
dev_err(adapter->pdev_dev, "bad SGE CPL MODE\n");
return -EINVAL;
}
/* Grab ingress packing boundary from SGE_CONTROL2 for */
params[0] = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_REG) |
V_FW_PARAMS_PARAM_XYZ(A_SGE_CONTROL2));
v = t4vf_query_params(adap, 1, params, vals);
if (v != FW_SUCCESS) {
dev_err(adapter, "Unable to get SGE Control2; "
"probably old firmware.\n");
return v;
}
sge_control2 = vals[0];
params[0] = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_REG) |
V_FW_PARAMS_PARAM_XYZ(A_SGE_INGRESS_RX_THRESHOLD));
params[1] = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_REG) |
V_FW_PARAMS_PARAM_XYZ(A_SGE_CONM_CTRL));
v = t4vf_query_params(adap, 2, params, vals);
if (v != FW_SUCCESS)
return v;
sge_ingress_rx_threshold = vals[0];
sge_congestion_control = vals[1];
params[0] = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_REG) |
V_FW_PARAMS_PARAM_XYZ(A_SGE_EGRESS_QUEUES_PER_PAGE_VF));
params[1] = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_REG) |
V_FW_PARAMS_PARAM_XYZ(A_SGE_INGRESS_QUEUES_PER_PAGE_VF));
v = t4vf_query_params(adap, 2, params, vals);
if (v != FW_SUCCESS) {
dev_warn(adap, "Unable to get VF SGE Queues/Page; "
"probably old firmware.\n");
return v;
}
sge_egress_queues_per_page = vals[0];
sge_ingress_queues_per_page = vals[1];
/*
* We need the Queues/Page for our VF. This is based on the
* PF from which we're instantiated and is indexed in the
* register we just read.
*/
s_hps = (S_HOSTPAGESIZEPF0 +
(S_HOSTPAGESIZEPF1 - S_HOSTPAGESIZEPF0) * adap->pf);
sge_params->hps =
((sge_host_page_size >> s_hps) & M_HOSTPAGESIZEPF0);
s_qpp = (S_QUEUESPERPAGEPF0 +
(S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * adap->pf);
sge_params->eq_qpp =
((sge_egress_queues_per_page >> s_qpp)
& M_QUEUESPERPAGEPF0);
sge_params->iq_qpp =
((sge_ingress_queues_per_page >> s_qpp)
& M_QUEUESPERPAGEPF0);
/*
* Now translate the queried parameters into our internal forms.
*/
if (fl_large_pg)
s->fl_pg_order = ilog2(fl_large_pg) - PAGE_SHIFT;
s->stat_len = ((sge_control & F_EGRSTATUSPAGESIZE)
? 128 : 64);
s->pktshift = G_PKTSHIFT(sge_control);
s->fl_align = t4vf_fl_pkt_align(adap, sge_control, sge_control2);
/*
* A FL with <= fl_starve_thres buffers is starving and a periodic
* timer will attempt to refill it. This needs to be larger than the
* SGE's Egress Congestion Threshold. If it isn't, then we can get
* stuck waiting for new packets while the SGE is waiting for us to
* give it more Free List entries. (Note that the SGE's Egress
* Congestion Threshold is in units of 2 Free List pointers.)
*/
switch (CHELSIO_CHIP_VERSION(adap->params.chip)) {
case CHELSIO_T5:
s->fl_starve_thres =
G_EGRTHRESHOLDPACKING(sge_congestion_control);
break;
case CHELSIO_T6:
default:
s->fl_starve_thres =
G_T6_EGRTHRESHOLDPACKING(sge_congestion_control);
break;
}
s->fl_starve_thres = s->fl_starve_thres * 2 + 1;
/*
* Save RX interrupt holdoff timer values and counter
* threshold values from the SGE parameters.
*/
s->timer_val[0] = core_ticks_to_us(adap,
G_TIMERVALUE0(sge_timer_value_0_and_1));
s->timer_val[1] = core_ticks_to_us(adap,
G_TIMERVALUE1(sge_timer_value_0_and_1));
s->timer_val[2] = core_ticks_to_us(adap,
G_TIMERVALUE2(sge_timer_value_2_and_3));
s->timer_val[3] = core_ticks_to_us(adap,
G_TIMERVALUE3(sge_timer_value_2_and_3));
s->timer_val[4] = core_ticks_to_us(adap,
G_TIMERVALUE4(sge_timer_value_4_and_5));
s->timer_val[5] = core_ticks_to_us(adap,
G_TIMERVALUE5(sge_timer_value_4_and_5));
s->counter_val[0] = G_THRESHOLD_0(sge_ingress_rx_threshold);
s->counter_val[1] = G_THRESHOLD_1(sge_ingress_rx_threshold);
s->counter_val[2] = G_THRESHOLD_2(sge_ingress_rx_threshold);
s->counter_val[3] = G_THRESHOLD_3(sge_ingress_rx_threshold);
return 0;
}
Reference in New Issue View Git Blame Copy Permalink