freebsd-dev/sys/dev/sfxge/common/efx_tx.c

/*-
 * Copyright (c) 2007-2015 Solarflare Communications Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * The views and conclusions contained in the software and documentation are
 * those of the authors and should not be interpreted as representing official
 * policies, either expressed or implied, of the FreeBSD Project.
 */

#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

#include "efsys.h"
#include "efx.h"
#include "efx_types.h"
#include "efx_regs.h"
#include "efx_impl.h"

#if EFSYS_OPT_QSTATS
#define	EFX_TX_QSTAT_INCR(_etp, _stat)					\
	do {								\
		(_etp)->et_stat[_stat]++;				\
	_NOTE(CONSTANTCONDITION)					\
	} while (B_FALSE)
#else
#define	EFX_TX_QSTAT_INCR(_etp, _stat)
#endif

#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA

static	__checkReturn	efx_rc_t
falconsiena_tx_init(
	__in		efx_nic_t *enp);

static			void
falconsiena_tx_fini(
	__in		efx_nic_t *enp);

static	__checkReturn	efx_rc_t
falconsiena_tx_qcreate(
	__in		efx_nic_t *enp,
	__in		unsigned int index,
	__in		unsigned int label,
	__in		efsys_mem_t *esmp,
	__in		size_t n,
	__in		uint32_t id,
	__in		uint16_t flags,
	__in		efx_evq_t *eep,
	__in		efx_txq_t *etp,
	__out		unsigned int *addedp);

static		void
falconsiena_tx_qdestroy(
	__in	efx_txq_t *etp);

static	__checkReturn	efx_rc_t
falconsiena_tx_qpost(
	__in		efx_txq_t *etp,
	__in_ecount(n)	efx_buffer_t *eb,
	__in		unsigned int n,
	__in		unsigned int completed,
	__inout		unsigned int *addedp);

static			void
falconsiena_tx_qpush(
	__in	efx_txq_t *etp,
	__in	unsigned int added,
	__in	unsigned int pushed);

static	__checkReturn	efx_rc_t
falconsiena_tx_qpace(
	__in		efx_txq_t *etp,
	__in		unsigned int ns);

static	__checkReturn	efx_rc_t
falconsiena_tx_qflush(
	__in		efx_txq_t *etp);

static			void
falconsiena_tx_qenable(
	__in	efx_txq_t *etp);

	__checkReturn	efx_rc_t
falconsiena_tx_qdesc_post(
	__in		efx_txq_t *etp,
	__in_ecount(n)	efx_desc_t *ed,
	__in		unsigned int n,
	__in		unsigned int completed,
	__inout		unsigned int *addedp);

	void
falconsiena_tx_qdesc_dma_create(
	__in	efx_txq_t *etp,
	__in	efsys_dma_addr_t addr,
	__in	size_t size,
	__in	boolean_t eop,
	__out	efx_desc_t *edp);

#if EFSYS_OPT_QSTATS
static			void
falconsiena_tx_qstats_update(
	__in				efx_txq_t *etp,
	__inout_ecount(TX_NQSTATS)	efsys_stat_t *stat);
#endif

#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */


#if EFSYS_OPT_FALCON
static efx_tx_ops_t	__efx_tx_falcon_ops = {
	falconsiena_tx_init,			/* etxo_init */
	falconsiena_tx_fini,			/* etxo_fini */
	falconsiena_tx_qcreate,			/* etxo_qcreate */
	falconsiena_tx_qdestroy,		/* etxo_qdestroy */
	falconsiena_tx_qpost,			/* etxo_qpost */
	falconsiena_tx_qpush,			/* etxo_qpush */
	falconsiena_tx_qpace,			/* etxo_qpace */
	falconsiena_tx_qflush,			/* etxo_qflush */
	falconsiena_tx_qenable,			/* etxo_qenable */
	NULL,					/* etxo_qpio_enable */
	NULL,					/* etxo_qpio_disable */
	NULL,					/* etxo_qpio_write */
	NULL,					/* etxo_qpio_post */
	falconsiena_tx_qdesc_post,		/* etxo_qdesc_post */
	falconsiena_tx_qdesc_dma_create,	/* etxo_qdesc_dma_create */
	NULL,					/* etxo_qdesc_tso_create */
	NULL,					/* etxo_qdesc_vlantci_create */
#if EFSYS_OPT_QSTATS
	falconsiena_tx_qstats_update,		/* etxo_qstats_update */
#endif
};
#endif /* EFSYS_OPT_FALCON */

#if EFSYS_OPT_SIENA
static efx_tx_ops_t	__efx_tx_siena_ops = {
	falconsiena_tx_init,			/* etxo_init */
	falconsiena_tx_fini,			/* etxo_fini */
	falconsiena_tx_qcreate,			/* etxo_qcreate */
	falconsiena_tx_qdestroy,		/* etxo_qdestroy */
	falconsiena_tx_qpost,			/* etxo_qpost */
	falconsiena_tx_qpush,			/* etxo_qpush */
	falconsiena_tx_qpace,			/* etxo_qpace */
	falconsiena_tx_qflush,			/* etxo_qflush */
	falconsiena_tx_qenable,			/* etxo_qenable */
	NULL,					/* etxo_qpio_enable */
	NULL,					/* etxo_qpio_disable */
	NULL,					/* etxo_qpio_write */
	NULL,					/* etxo_qpio_post */
	falconsiena_tx_qdesc_post,		/* etxo_qdesc_post */
	falconsiena_tx_qdesc_dma_create,	/* etxo_qdesc_dma_create */
	NULL,					/* etxo_qdesc_tso_create */
	NULL,					/* etxo_qdesc_vlantci_create */
#if EFSYS_OPT_QSTATS
	falconsiena_tx_qstats_update,		/* etxo_qstats_update */
#endif
};
#endif /* EFSYS_OPT_SIENA */

#if EFSYS_OPT_HUNTINGTON
static efx_tx_ops_t	__efx_tx_hunt_ops = {
	hunt_tx_init,				/* etxo_init */
	hunt_tx_fini,				/* etxo_fini */
	hunt_tx_qcreate,			/* etxo_qcreate */
	hunt_tx_qdestroy,			/* etxo_qdestroy */
	hunt_tx_qpost,				/* etxo_qpost */
	hunt_tx_qpush,				/* etxo_qpush */
	hunt_tx_qpace,				/* etxo_qpace */
	hunt_tx_qflush,				/* etxo_qflush */
	hunt_tx_qenable,			/* etxo_qenable */
	hunt_tx_qpio_enable,			/* etxo_qpio_enable */
	hunt_tx_qpio_disable,			/* etxo_qpio_disable */
	hunt_tx_qpio_write,			/* etxo_qpio_write */
	hunt_tx_qpio_post,			/* etxo_qpio_post */
	hunt_tx_qdesc_post,			/* etxo_qdesc_post */
	hunt_tx_qdesc_dma_create,		/* etxo_qdesc_dma_create */
	hunt_tx_qdesc_tso_create,		/* etxo_qdesc_tso_create */
	hunt_tx_qdesc_vlantci_create,		/* etxo_qdesc_vlantci_create */
#if EFSYS_OPT_QSTATS
	hunt_tx_qstats_update,			/* etxo_qstats_update */
#endif
};
#endif /* EFSYS_OPT_HUNTINGTON */

	__checkReturn	efx_rc_t
efx_tx_init(
	__in		efx_nic_t *enp)
{
	efx_tx_ops_t *etxop;
	efx_rc_t rc;

	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);

	if (!(enp->en_mod_flags & EFX_MOD_EV)) {
		rc = EINVAL;
		goto fail1;
	}

	if (enp->en_mod_flags & EFX_MOD_TX) {
		rc = EINVAL;
		goto fail2;
	}

	switch (enp->en_family) {
#if EFSYS_OPT_FALCON
	case EFX_FAMILY_FALCON:
		etxop = (efx_tx_ops_t *)&__efx_tx_falcon_ops;
		break;
#endif /* EFSYS_OPT_FALCON */

#if EFSYS_OPT_SIENA
	case EFX_FAMILY_SIENA:
		etxop = (efx_tx_ops_t *)&__efx_tx_siena_ops;
		break;
#endif /* EFSYS_OPT_SIENA */

#if EFSYS_OPT_HUNTINGTON
	case EFX_FAMILY_HUNTINGTON:
		etxop = (efx_tx_ops_t *)&__efx_tx_hunt_ops;
		break;
#endif /* EFSYS_OPT_HUNTINGTON */

	default:
		EFSYS_ASSERT(0);
		rc = ENOTSUP;
		goto fail3;
	}

	EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);

	if ((rc = etxop->etxo_init(enp)) != 0)
		goto fail4;

	enp->en_etxop = etxop;
	enp->en_mod_flags |= EFX_MOD_TX;
	return (0);

fail4:
	EFSYS_PROBE(fail4);
fail3:
	EFSYS_PROBE(fail3);
fail2:
	EFSYS_PROBE(fail2);
fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);

	enp->en_etxop = NULL;
	enp->en_mod_flags &= ~EFX_MOD_TX;
	return (rc);
}

			void
efx_tx_fini(
	__in	efx_nic_t *enp)
{
	efx_tx_ops_t *etxop = enp->en_etxop;

	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
	EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);

	etxop->etxo_fini(enp);

	enp->en_etxop = NULL;
	enp->en_mod_flags &= ~EFX_MOD_TX;
}

	__checkReturn	efx_rc_t
efx_tx_qcreate(
	__in		efx_nic_t *enp,
	__in		unsigned int index,
	__in		unsigned int label,
	__in		efsys_mem_t *esmp,
	__in		size_t n,
	__in		uint32_t id,
	__in		uint16_t flags,
	__in		efx_evq_t *eep,
	__deref_out	efx_txq_t **etpp,
	__out		unsigned int *addedp)
{
	efx_tx_ops_t *etxop = enp->en_etxop;
	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
	efx_txq_t *etp;
	efx_rc_t rc;

	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
	EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);

	EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit);

	/* Allocate an TXQ object */
	EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);

	if (etp == NULL) {
		rc = ENOMEM;
		goto fail1;
	}

	etp->et_magic = EFX_TXQ_MAGIC;
	etp->et_enp = enp;
	etp->et_index = index;
	etp->et_mask = n - 1;
	etp->et_esmp = esmp;

	/* Initial descriptor index may be modified by etxo_qcreate */
	*addedp = 0;

	if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
	    n, id, flags, eep, etp, addedp)) != 0)
			goto fail2;

	enp->en_tx_qcount++;
	*etpp = etp;

	return (0);

fail2:
	EFSYS_PROBE(fail2);
	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	return (rc);
}

		void
efx_tx_qdestroy(
	__in	efx_txq_t *etp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

	EFSYS_ASSERT(enp->en_tx_qcount != 0);
	--enp->en_tx_qcount;

	etxop->etxo_qdestroy(etp);

	/* Free the TXQ object */
	EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
}

	__checkReturn	efx_rc_t
efx_tx_qpost(
	__in		efx_txq_t *etp,
	__in_ecount(n)	efx_buffer_t *eb,
	__in		unsigned int n,
	__in		unsigned int completed,
	__inout		unsigned int *addedp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;
	efx_rc_t rc;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

	if ((rc = etxop->etxo_qpost(etp, eb,
	    n, completed, addedp)) != 0)
		goto fail1;

	return (0);

fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	return (rc);
}

			void
efx_tx_qpush(
	__in	efx_txq_t *etp,
	__in	unsigned int added,
	__in	unsigned int pushed)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

	etxop->etxo_qpush(etp, added, pushed);
}

	__checkReturn	efx_rc_t
efx_tx_qpace(
	__in		efx_txq_t *etp,
	__in		unsigned int ns)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;
	efx_rc_t rc;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

	if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
		goto fail1;

	return (0);

fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	return (rc);
}

	__checkReturn	efx_rc_t
efx_tx_qflush(
	__in	efx_txq_t *etp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;
	efx_rc_t rc;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

	if ((rc = etxop->etxo_qflush(etp)) != 0)
		goto fail1;

	return (0);

fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	return (rc);
}

			void
efx_tx_qenable(
	__in	efx_txq_t *etp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

	etxop->etxo_qenable(etp);
}

	__checkReturn	efx_rc_t
efx_tx_qpio_enable(
	__in	efx_txq_t *etp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;
	efx_rc_t rc;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

	if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
		rc = ENOTSUP;
		goto fail1;
	}
	if (etxop->etxo_qpio_enable == NULL) {
		rc = ENOTSUP;
		goto fail2;
	}
	if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
		goto fail3;

	return (0);

fail3:
	EFSYS_PROBE(fail3);
fail2:
	EFSYS_PROBE(fail2);
fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	return (rc);
}

		void
efx_tx_qpio_disable(
	__in	efx_txq_t *etp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

	if (etxop->etxo_qpio_disable != NULL)
		etxop->etxo_qpio_disable(etp);
}

	__checkReturn	efx_rc_t
efx_tx_qpio_write(
	__in			efx_txq_t *etp,
	__in_ecount(buf_length)	uint8_t *buffer,
	__in			size_t buf_length,
	__in			size_t pio_buf_offset)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;
	efx_rc_t rc;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

	if (etxop->etxo_qpio_write != NULL) {
		if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
						pio_buf_offset)) != 0)
			goto fail1;
		return (0);
	}

	return (ENOTSUP);

fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	return (rc);
}

	__checkReturn	efx_rc_t
efx_tx_qpio_post(
	__in			efx_txq_t *etp,
	__in			size_t pkt_length,
	__in			unsigned int completed,
	__inout			unsigned int *addedp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;
	efx_rc_t rc;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

	if (etxop->etxo_qpio_post != NULL) {
		if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
						addedp)) != 0)
			goto fail1;
		return (0);
	}

	return (ENOTSUP);

fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	return (rc);
}

	__checkReturn	efx_rc_t
efx_tx_qdesc_post(
	__in		efx_txq_t *etp,
	__in_ecount(n)	efx_desc_t *ed,
	__in		unsigned int n,
	__in		unsigned int completed,
	__inout		unsigned int *addedp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;
	efx_rc_t rc;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

	if ((rc = etxop->etxo_qdesc_post(etp, ed,
	    n, completed, addedp)) != 0)
		goto fail1;

	return (0);

fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	return (rc);
}

	void
efx_tx_qdesc_dma_create(
	__in	efx_txq_t *etp,
	__in	efsys_dma_addr_t addr,
	__in	size_t size,
	__in	boolean_t eop,
	__out	efx_desc_t *edp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
	EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);

	etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
}

	void
efx_tx_qdesc_tso_create(
	__in	efx_txq_t *etp,
	__in	uint16_t ipv4_id,
	__in	uint32_t tcp_seq,
	__in	uint8_t  tcp_flags,
	__out	efx_desc_t *edp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
	EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);

	etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
}

	void
efx_tx_qdesc_vlantci_create(
	__in	efx_txq_t *etp,
	__in	uint16_t tci,
	__out	efx_desc_t *edp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
	EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);

	etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
}


#if EFSYS_OPT_QSTATS
			void
efx_tx_qstats_update(
	__in				efx_txq_t *etp,
	__inout_ecount(TX_NQSTATS)	efsys_stat_t *stat)
{
	efx_nic_t *enp = etp->et_enp;
	efx_tx_ops_t *etxop = enp->en_etxop;

	EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);

	etxop->etxo_qstats_update(etp, stat);
}
#endif


#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA

static	__checkReturn	efx_rc_t
falconsiena_tx_init(
	__in		efx_nic_t *enp)
{
	efx_oword_t oword;

	/*
	 * Disable the timer-based TX DMA backoff and allow TX DMA to be
	 * controlled by the RX FIFO fill level (although always allow a
	 * minimal trickle).
	 */
	EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);

	/*
	 * Filter all packets less than 14 bytes to avoid parsing
	 * errors.
	 */
	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
	EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);

	/*
	 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
	 * descriptors (which is bad).
	 */
	EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
	EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);

	return (0);
}

#define	EFX_TX_DESC(_etp, _addr, _size, _eop, _added)			\
	do {								\
		unsigned int id;					\
		size_t offset;						\
		efx_qword_t qword;					\
									\
		id = (_added)++ & (_etp)->et_mask;			\
		offset = id * sizeof (efx_qword_t);			\
									\
		EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index,	\
		    unsigned int, id, efsys_dma_addr_t, (_addr),	\
		    size_t, (_size), boolean_t, (_eop));		\
									\
		EFX_POPULATE_QWORD_4(qword,				\
		    FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1,			\
		    FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size),	\
		    FSF_AZ_TX_KER_BUF_ADDR_DW0,				\
		    (uint32_t)((_addr) & 0xffffffff),			\
		    FSF_AZ_TX_KER_BUF_ADDR_DW1,				\
		    (uint32_t)((_addr) >> 32));				\
		EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword);	\
									\
		_NOTE(CONSTANTCONDITION)				\
	} while (B_FALSE)

static	__checkReturn	efx_rc_t
falconsiena_tx_qpost(
	__in		efx_txq_t *etp,
	__in_ecount(n)	efx_buffer_t *eb,
	__in		unsigned int n,
	__in		unsigned int completed,
	__inout		unsigned int *addedp)
{
	unsigned int added = *addedp;
	unsigned int i;
	int rc = ENOSPC;

	if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
		goto fail1;

	for (i = 0; i < n; i++) {
		efx_buffer_t *ebp = &eb[i];
		efsys_dma_addr_t start = ebp->eb_addr;
		size_t size = ebp->eb_size;
		efsys_dma_addr_t end = start + size;

		/* Fragments must not span 4k boundaries. */
		EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end);

		EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
	}

	EFX_TX_QSTAT_INCR(etp, TX_POST);

	*addedp = added;
	return (0);

fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);

	return (rc);
}

static		void
falconsiena_tx_qpush(
	__in	efx_txq_t *etp,
	__in	unsigned int added,
	__in	unsigned int pushed)
{
	efx_nic_t *enp = etp->et_enp;
	uint32_t wptr;
	efx_dword_t dword;
	efx_oword_t oword;

	/* Push the populated descriptors out */
	wptr = added & etp->et_mask;

	EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);

	/* Only write the third DWORD */
	EFX_POPULATE_DWORD_1(dword,
	    EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));

	/* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
	EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
	    wptr, pushed & etp->et_mask);
	EFSYS_PIO_WRITE_BARRIER();
	EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
			    etp->et_index, &dword, B_FALSE);
}

#define	EFX_MAX_PACE_VALUE 20
#define	EFX_TX_PACE_CLOCK_BASE	104

static	__checkReturn	efx_rc_t
falconsiena_tx_qpace(
	__in		efx_txq_t *etp,
	__in		unsigned int ns)
{
	efx_nic_t *enp = etp->et_enp;
	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
	efx_oword_t oword;
	unsigned int pace_val;
	unsigned int timer_period;
	efx_rc_t rc;

	if (ns == 0) {
		pace_val = 0;
	} else {
		/*
		 * The pace_val to write into the table is s.t
		 * ns <= timer_period * (2 ^ pace_val)
		 */
		timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult;
		for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
			if ((timer_period << pace_val) >= ns)
				break;
		}
	}
	if (pace_val > EFX_MAX_PACE_VALUE) {
		rc = EINVAL;
		goto fail1;
	}

	/* Update the pacing table */
	EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
	    &oword, B_TRUE);

	return (0);

fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);

	return (rc);
}

static	__checkReturn	efx_rc_t
falconsiena_tx_qflush(
	__in		efx_txq_t *etp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_oword_t oword;
	uint32_t label;

	efx_tx_qpace(etp, 0);

	label = etp->et_index;

	/* Flush the queue */
	EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
	    FRF_AZ_TX_FLUSH_DESCQ, label);
	EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);

	return (0);
}

static		void
falconsiena_tx_qenable(
	__in	efx_txq_t *etp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_oword_t oword;

	EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
			    etp->et_index, &oword, B_TRUE);

	EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
	    uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));

	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
	EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);

	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
			    etp->et_index, &oword, B_TRUE);
}

static	__checkReturn	efx_rc_t
falconsiena_tx_qcreate(
	__in		efx_nic_t *enp,
	__in		unsigned int index,
	__in		unsigned int label,
	__in		efsys_mem_t *esmp,
	__in		size_t n,
	__in		uint32_t id,
	__in		uint16_t flags,
	__in		efx_evq_t *eep,
	__in		efx_txq_t *etp,
	__out		unsigned int *addedp)
{
	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
	efx_oword_t oword;
	uint32_t size;
	efx_rc_t rc;

	EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
	    (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
	EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);

	EFSYS_ASSERT(ISP2(EFX_TXQ_MAXNDESCS(encp)));
	EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));

	if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
		rc = EINVAL;
		goto fail1;
	}
	if (index >= encp->enc_txq_limit) {
		rc = EINVAL;
		goto fail2;
	}
	for (size = 0;
	    (1 << size) <= (EFX_TXQ_MAXNDESCS(encp) / EFX_TXQ_MINNDESCS);
	    size++)
		if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
			break;
	if (id + (1 << size) >= encp->enc_buftbl_limit) {
		rc = EINVAL;
		goto fail3;
	}

	/* Set up the new descriptor queue */
	*addedp = 0;

	EFX_POPULATE_OWORD_6(oword,
	    FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
	    FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
	    FRF_AZ_TX_DESCQ_OWNER_ID, 0,
	    FRF_AZ_TX_DESCQ_LABEL, label,
	    FRF_AZ_TX_DESCQ_SIZE, size,
	    FRF_AZ_TX_DESCQ_TYPE, 0);

	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
	    (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
	EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
	    (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);

	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
	    etp->et_index, &oword, B_TRUE);

	return (0);

fail3:
	EFSYS_PROBE(fail3);
fail2:
	EFSYS_PROBE(fail2);
fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);

	return (rc);
}

	__checkReturn	efx_rc_t
falconsiena_tx_qdesc_post(
	__in		efx_txq_t *etp,
	__in_ecount(n)	efx_desc_t *ed,
	__in		unsigned int n,
	__in		unsigned int completed,
	__inout		unsigned int *addedp)
{
	unsigned int added = *addedp;
	unsigned int i;
	efx_rc_t rc;

	if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
		rc = ENOSPC;
		goto fail1;
	}

	for (i = 0; i < n; i++) {
		efx_desc_t *edp = &ed[i];
		unsigned int id;
		size_t offset;

		id = added++ & etp->et_mask;
		offset = id * sizeof (efx_desc_t);

		EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
	}

	EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
		    unsigned int, added, unsigned int, n);

	EFX_TX_QSTAT_INCR(etp, TX_POST);

	*addedp = added;
	return (0);

fail1:
	EFSYS_PROBE1(fail1, efx_rc_t, rc);
	return (rc);
}

	void
falconsiena_tx_qdesc_dma_create(
	__in	efx_txq_t *etp,
	__in	efsys_dma_addr_t addr,
	__in	size_t size,
	__in	boolean_t eop,
	__out	efx_desc_t *edp)
{
	/* Fragments must not span 4k boundaries. */
	EFSYS_ASSERT(P2ROUNDUP(addr + 1, 4096) >= addr + size);

	EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
		    efsys_dma_addr_t, addr,
		    size_t, size, boolean_t, eop);

	EFX_POPULATE_QWORD_4(edp->ed_eq,
			    FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
			    FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
			    FSF_AZ_TX_KER_BUF_ADDR_DW0,
			    (uint32_t)(addr & 0xffffffff),
			    FSF_AZ_TX_KER_BUF_ADDR_DW1,
			    (uint32_t)(addr >> 32));
}

#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */

#if EFSYS_OPT_QSTATS
#if EFSYS_OPT_NAMES
/* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 9d8d26a0a5e2c453 */
static const char 	*__efx_tx_qstat_name[] = {
	"post",
	"post_pio",
};
/* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */

		const char *
efx_tx_qstat_name(
	__in	efx_nic_t *enp,
	__in	unsigned int id)
{
	_NOTE(ARGUNUSED(enp))
	EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
	EFSYS_ASSERT3U(id, <, TX_NQSTATS);

	return (__efx_tx_qstat_name[id]);
}
#endif	/* EFSYS_OPT_NAMES */
#endif /* EFSYS_OPT_QSTATS */

#if EFSYS_OPT_FALCON || EFSYS_OPT_SIENA

#if EFSYS_OPT_QSTATS
static					void
falconsiena_tx_qstats_update(
	__in				efx_txq_t *etp,
	__inout_ecount(TX_NQSTATS)	efsys_stat_t *stat)
{
	unsigned int id;

	for (id = 0; id < TX_NQSTATS; id++) {
		efsys_stat_t *essp = &stat[id];

		EFSYS_STAT_INCR(essp, etp->et_stat[id]);
		etp->et_stat[id] = 0;
	}
}
#endif	/* EFSYS_OPT_QSTATS */

static		void
falconsiena_tx_qdestroy(
	__in	efx_txq_t *etp)
{
	efx_nic_t *enp = etp->et_enp;
	efx_oword_t oword;

	/* Purge descriptor queue */
	EFX_ZERO_OWORD(oword);

	EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
			    etp->et_index, &oword, B_TRUE);
}

static		void
falconsiena_tx_fini(
	__in	efx_nic_t *enp)
{
	_NOTE(ARGUNUSED(enp))
}

#endif /* EFSYS_OPT_FALCON || EFSYS_OPT_SIENA */