numam-dpdk/drivers/net/mpipe/mpipe_tilegx.c

/*-
 *   BSD LICENSE
 *
 *   Copyright(c) 2015 EZchip Semiconductor Ltd. All rights reserved.
 *
 *   Redistribution and use in source and binary forms, with or without
 *   modification, are permitted provided that the following conditions
 *   are met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in
 *       the documentation and/or other materials provided with the
 *       distribution.
 *     * Neither the name of EZchip Semiconductor nor the names of its
 *       contributors may be used to endorse or promote products derived
 *       from this software without specific prior written permission.
 *
 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <unistd.h>

#include <rte_eal.h>
#include <rte_dev.h>
#include <rte_eal_memconfig.h>
#include <rte_ethdev.h>
#include <rte_malloc.h>
#include <rte_cycles.h>

#include <arch/mpipe_xaui_def.h>
#include <arch/mpipe_gbe_def.h>

#include <gxio/mpipe.h>

#ifdef RTE_LIBRTE_MPIPE_PMD_DEBUG
#define PMD_DEBUG_RX(...)	RTE_LOG(DEBUG, PMD, __VA_ARGS__)
#define PMD_DEBUG_TX(...)	RTE_LOG(DEBUG, PMD, __VA_ARGS__)
#else
#define PMD_DEBUG_RX(...)
#define PMD_DEBUG_TX(...)
#endif

#define MPIPE_MAX_CHANNELS		128
#define MPIPE_TX_MAX_QUEUES		128
#define MPIPE_RX_MAX_QUEUES		16
#define MPIPE_TX_DESCS			512
#define MPIPE_RX_BUCKETS		256
#define MPIPE_RX_STACK_SIZE		65536
#define MPIPE_RX_IP_ALIGN		2
#define MPIPE_BSM_ALIGN			128

#define MPIPE_LINK_UPDATE_TIMEOUT	10	/*  s */
#define MPIPE_LINK_UPDATE_INTERVAL	100000	/* us */

struct mpipe_channel_config {
	int enable;
	int first_bucket;
	int num_buckets;
	int head_room;
	gxio_mpipe_rules_stacks_t stacks;
};

struct mpipe_context {
	rte_spinlock_t        lock;
	gxio_mpipe_context_t  context;
	struct mpipe_channel_config channels[MPIPE_MAX_CHANNELS];
};

/* Per-core local data. */
struct mpipe_local {
	int mbuf_push_debt[RTE_MAX_ETHPORTS];	/* Buffer push debt. */
} __rte_cache_aligned;

#define MPIPE_BUF_DEBT_THRESHOLD	32
static __thread struct mpipe_local mpipe_local;
static struct mpipe_context mpipe_contexts[GXIO_MPIPE_INSTANCE_MAX];
static int mpipe_instances;
static const char *drivername = "MPIPE PMD";

/* Per queue statistics. */
struct mpipe_queue_stats {
	uint64_t packets, bytes, errors, nomem;
};

/* Common tx/rx queue fields. */
struct mpipe_queue {
	struct mpipe_dev_priv *priv;	/* "priv" data of its device. */
	uint16_t nb_desc;		/* Number of tx descriptors. */
	uint16_t port_id;		/* Device index. */
	uint16_t stat_idx;		/* Queue stats index. */
	uint8_t queue_idx;		/* Queue index. */
	uint8_t link_status;		/* 0 = link down. */
	struct mpipe_queue_stats stats;	/* Stat data for the queue. */
};

/* Transmit queue description. */
struct mpipe_tx_queue {
	struct mpipe_queue q;		/* Common stuff. */
};

/* Receive queue description. */
struct mpipe_rx_queue {
	struct mpipe_queue q;		/* Common stuff. */
	gxio_mpipe_iqueue_t iqueue;	/* mPIPE iqueue. */
	gxio_mpipe_idesc_t *next_desc;	/* Next idesc to process. */
	int avail_descs;		/* Number of available descs. */
	void *rx_ring_mem;		/* DMA ring memory. */
};

struct mpipe_dev_priv {
	gxio_mpipe_context_t *context;	/* mPIPE context. */
	gxio_mpipe_link_t link;		/* mPIPE link for the device. */
	gxio_mpipe_equeue_t equeue;	/* mPIPE equeue. */
	unsigned equeue_size;		/* mPIPE equeue desc count. */
	int instance;			/* mPIPE instance. */
	int ering;			/* mPIPE eDMA ring. */
	int stack;			/* mPIPE buffer stack. */
	int channel;			/* Device channel. */
	int port_id;			/* DPDK port index. */
	struct rte_eth_dev *eth_dev;	/* DPDK device. */
	struct rte_mbuf **tx_comps;	/* TX completion array. */
	struct rte_mempool *rx_mpool;	/* mpool used by the rx queues. */
	unsigned rx_offset;		/* Receive head room. */
	unsigned rx_size_code;		/* mPIPE rx buffer size code. */
	int is_xaui:1,			/* Is this an xgbe or gbe? */
	    initialized:1,		/* Initialized port? */
	    running:1;			/* Running port? */
	struct ether_addr mac_addr;	/* MAC address. */
	unsigned nb_rx_queues;		/* Configured tx queues. */
	unsigned nb_tx_queues;		/* Configured rx queues. */
	int first_bucket;		/* mPIPE bucket start index. */
	int first_ring;			/* mPIPE notif ring start index. */
	int notif_group;		/* mPIPE notif group. */
	rte_atomic32_t dp_count __rte_cache_aligned;	/* DP Entry count. */
	int tx_stat_mapping[RTE_ETHDEV_QUEUE_STAT_CNTRS];
	int rx_stat_mapping[RTE_ETHDEV_QUEUE_STAT_CNTRS];
};

#define mpipe_priv(dev)			\
	((struct mpipe_dev_priv*)(dev)->data->dev_private)

#define mpipe_name(priv)		\
	((priv)->eth_dev->data->name)

#define mpipe_rx_queue(priv, n)		\
	((struct mpipe_rx_queue *)(priv)->eth_dev->data->rx_queues[n])

#define mpipe_tx_queue(priv, n)		\
	((struct mpipe_tx_queue *)(priv)->eth_dev->data->tx_queues[n])

static void
mpipe_xmit_flush(struct mpipe_dev_priv *priv);

static void
mpipe_recv_flush(struct mpipe_dev_priv *priv);

static int mpipe_equeue_sizes[] = {
	[GXIO_MPIPE_EQUEUE_ENTRY_512]	= 512,
	[GXIO_MPIPE_EQUEUE_ENTRY_2K]	= 2048,
	[GXIO_MPIPE_EQUEUE_ENTRY_8K]	= 8192,
	[GXIO_MPIPE_EQUEUE_ENTRY_64K]	= 65536,
};

static int mpipe_iqueue_sizes[] = {
	[GXIO_MPIPE_IQUEUE_ENTRY_128]	= 128,
	[GXIO_MPIPE_IQUEUE_ENTRY_512]	= 512,
	[GXIO_MPIPE_IQUEUE_ENTRY_2K]	= 2048,
	[GXIO_MPIPE_IQUEUE_ENTRY_64K]	= 65536,
};

static int mpipe_buffer_sizes[] = {
	[GXIO_MPIPE_BUFFER_SIZE_128]	= 128,
	[GXIO_MPIPE_BUFFER_SIZE_256]	= 256,
	[GXIO_MPIPE_BUFFER_SIZE_512]	= 512,
	[GXIO_MPIPE_BUFFER_SIZE_1024]	= 1024,
	[GXIO_MPIPE_BUFFER_SIZE_1664]	= 1664,
	[GXIO_MPIPE_BUFFER_SIZE_4096]	= 4096,
	[GXIO_MPIPE_BUFFER_SIZE_10368]	= 10368,
	[GXIO_MPIPE_BUFFER_SIZE_16384]	= 16384,
};

static gxio_mpipe_context_t *
mpipe_context(int instance)
{
	if (instance < 0 || instance >= mpipe_instances)
		return NULL;
	return &mpipe_contexts[instance].context;
}

static int mpipe_channel_config(int instance, int channel,
				struct mpipe_channel_config *config)
{
	struct mpipe_channel_config *data;
	struct mpipe_context *context;
	gxio_mpipe_rules_t rules;
	int idx, rc = 0;

	if (instance < 0 || instance >= mpipe_instances ||
	    channel < 0 || channel >= MPIPE_MAX_CHANNELS)
		return -EINVAL;

	context = &mpipe_contexts[instance];

	rte_spinlock_lock(&context->lock);

	gxio_mpipe_rules_init(&rules, &context->context);

	for (idx = 0; idx < MPIPE_MAX_CHANNELS; idx++) {
		data = (channel == idx) ? config : &context->channels[idx];

		if (!data->enable)
			continue;

		rc = gxio_mpipe_rules_begin(&rules, data->first_bucket,
					    data->num_buckets, &data->stacks);
		if (rc < 0) {
			goto done;
		}

		rc = gxio_mpipe_rules_add_channel(&rules, idx);
		if (rc < 0) {
			goto done;
		}

		rc = gxio_mpipe_rules_set_headroom(&rules, data->head_room);
		if (rc < 0) {
			goto done;
		}
	}

	rc = gxio_mpipe_rules_commit(&rules);
	if (rc == 0) {
		memcpy(&context->channels[channel], config, sizeof(*config));
	}

done:
	rte_spinlock_unlock(&context->lock);

	return rc;
}

static int
mpipe_get_size_index(int *array, int count, int size,
		     bool roundup)
{
	int i, last = -1;

	for (i = 0; i < count && array[i] < size; i++) {
		if (array[i])
			last = i;
	}

	if (roundup)
		return i < count ? (int)i : -ENOENT;
	else
		return last >= 0 ? last : -ENOENT;
}

static int
mpipe_calc_size(int *array, int count, int size)
{
	int index = mpipe_get_size_index(array, count, size, 1);
	return index < 0 ? index : array[index];
}

static int mpipe_equeue_size(int size)
{
	int result;
	result = mpipe_calc_size(mpipe_equeue_sizes,
				 RTE_DIM(mpipe_equeue_sizes), size);
	return result;
}

static int mpipe_iqueue_size(int size)
{
	int result;
	result = mpipe_calc_size(mpipe_iqueue_sizes,
				 RTE_DIM(mpipe_iqueue_sizes), size);
	return result;
}

static int mpipe_buffer_size_index(int size)
{
	int result;
	result = mpipe_get_size_index(mpipe_buffer_sizes,
				      RTE_DIM(mpipe_buffer_sizes), size, 0);
	return result;
}

static inline int
mpipe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
				  struct rte_eth_link *link)
{
	struct rte_eth_link *dst = link;
	struct rte_eth_link *src = &(dev->data->dev_link);

	if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
				*(uint64_t *)src) == 0)
		return -1;

	return 0;
}

static inline int
mpipe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
				   struct rte_eth_link *link)
{
	struct rte_eth_link *dst = &(dev->data->dev_link);
	struct rte_eth_link *src = link;

	if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
				*(uint64_t *)src) == 0)
		return -1;

	return 0;
}

static void
mpipe_infos_get(struct rte_eth_dev *dev __rte_unused,
		struct rte_eth_dev_info *dev_info)
{
	dev_info->min_rx_bufsize  = 128;
	dev_info->max_rx_pktlen   = 1518;
	dev_info->max_tx_queues   = MPIPE_TX_MAX_QUEUES;
	dev_info->max_rx_queues   = MPIPE_RX_MAX_QUEUES;
	dev_info->max_mac_addrs   = 1;
	dev_info->rx_offload_capa = 0;
	dev_info->tx_offload_capa = 0;
}

static int
mpipe_configure(struct rte_eth_dev *dev)
{
	struct mpipe_dev_priv *priv = mpipe_priv(dev);

	if (dev->data->nb_tx_queues > MPIPE_TX_MAX_QUEUES) {
		RTE_LOG(ERR, PMD, "%s: Too many tx queues: %d > %d\n",
			mpipe_name(priv), dev->data->nb_tx_queues,
			MPIPE_TX_MAX_QUEUES);
		return -EINVAL;
	}
	priv->nb_tx_queues = dev->data->nb_tx_queues;

	if (dev->data->nb_rx_queues > MPIPE_RX_MAX_QUEUES) {
		RTE_LOG(ERR, PMD, "%s: Too many rx queues: %d > %d\n",
			mpipe_name(priv), dev->data->nb_rx_queues,
			MPIPE_RX_MAX_QUEUES);
	}
	priv->nb_rx_queues = dev->data->nb_rx_queues;

	return 0;
}

static inline int
mpipe_link_compare(struct rte_eth_link *link1,
		   struct rte_eth_link *link2)
{
	return (*(uint64_t *)link1 == *(uint64_t *)link2)
		? -1 : 0;
}

static int
mpipe_link_update(struct rte_eth_dev *dev, int wait_to_complete)
{
	struct mpipe_dev_priv *priv = mpipe_priv(dev);
	struct rte_eth_link old, new;
	int64_t state, speed;
	int count, rc;

	memset(&old, 0, sizeof(old));
	memset(&new, 0, sizeof(new));
	mpipe_dev_atomic_read_link_status(dev, &old);

	for (count = 0, rc = 0; count < MPIPE_LINK_UPDATE_TIMEOUT; count++) {
		if (!priv->initialized)
			break;

		state = gxio_mpipe_link_get_attr(&priv->link,
						 GXIO_MPIPE_LINK_CURRENT_STATE);
		if (state < 0)
			break;

		speed = state & GXIO_MPIPE_LINK_SPEED_MASK;

		new.link_autoneg = (dev->data->dev_conf.link_speeds &
				ETH_LINK_SPEED_AUTONEG);
		if (speed == GXIO_MPIPE_LINK_1G) {
			new.link_speed = ETH_SPEED_NUM_1G;
			new.link_duplex = ETH_LINK_FULL_DUPLEX;
			new.link_status = ETH_LINK_UP;
		} else if (speed == GXIO_MPIPE_LINK_10G) {
			new.link_speed = ETH_SPEED_NUM_10G;
			new.link_duplex = ETH_LINK_FULL_DUPLEX;
			new.link_status = ETH_LINK_UP;
		}

		rc = mpipe_link_compare(&old, &new);
		if (rc == 0 || !wait_to_complete)
			break;

		rte_delay_us(MPIPE_LINK_UPDATE_INTERVAL);
	}

	mpipe_dev_atomic_write_link_status(dev, &new);
	return rc;
}

static int
mpipe_set_link(struct rte_eth_dev *dev, int up)
{
	struct mpipe_dev_priv *priv = mpipe_priv(dev);
	int rc;

	rc = gxio_mpipe_link_set_attr(&priv->link,
				      GXIO_MPIPE_LINK_DESIRED_STATE,
				      up ? GXIO_MPIPE_LINK_ANYSPEED : 0);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to set link %s.\n",
			mpipe_name(priv), up ? "up" : "down");
	} else {
		mpipe_link_update(dev, 0);
	}

	return rc;
}

static int
mpipe_set_link_up(struct rte_eth_dev *dev)
{
	return mpipe_set_link(dev, 1);
}

static int
mpipe_set_link_down(struct rte_eth_dev *dev)
{
	return mpipe_set_link(dev, 0);
}

static inline void
mpipe_dp_enter(struct mpipe_dev_priv *priv)
{
	__insn_mtspr(SPR_DSTREAM_PF, 0);
	rte_atomic32_inc(&priv->dp_count);
}

static inline void
mpipe_dp_exit(struct mpipe_dev_priv *priv)
{
	rte_atomic32_dec(&priv->dp_count);
}

static inline void
mpipe_dp_wait(struct mpipe_dev_priv *priv)
{
	while (rte_atomic32_read(&priv->dp_count) != 0) {
		rte_pause();
	}
}

static inline int
mpipe_mbuf_stack_index(struct mpipe_dev_priv *priv, struct rte_mbuf *mbuf)
{
	return (mbuf->port < RTE_MAX_ETHPORTS) ?
		mpipe_priv(&rte_eth_devices[mbuf->port])->stack :
		priv->stack;
}

static inline struct rte_mbuf *
mpipe_recv_mbuf(struct mpipe_dev_priv *priv, gxio_mpipe_idesc_t *idesc,
		int in_port)
{
	void *va = gxio_mpipe_idesc_get_va(idesc);
	uint16_t size = gxio_mpipe_idesc_get_xfer_size(idesc);
	struct rte_mbuf *mbuf = RTE_PTR_SUB(va, priv->rx_offset);

	rte_pktmbuf_reset(mbuf);
	mbuf->data_off = (uintptr_t)va - (uintptr_t)mbuf->buf_addr;
	mbuf->port     = in_port;
	mbuf->data_len = size;
	mbuf->pkt_len  = size;
	mbuf->hash.rss = gxio_mpipe_idesc_get_flow_hash(idesc);

	PMD_DEBUG_RX("%s: RX mbuf %p, buffer %p, buf_addr %p, size %d\n",
		     mpipe_name(priv), mbuf, va, mbuf->buf_addr, size);

	return mbuf;
}

static inline void
mpipe_recv_push(struct mpipe_dev_priv *priv, struct rte_mbuf *mbuf)
{
	const int offset = RTE_PKTMBUF_HEADROOM + MPIPE_RX_IP_ALIGN;
	void *buf_addr = RTE_PTR_ADD(mbuf->buf_addr, offset);

	gxio_mpipe_push_buffer(priv->context, priv->stack, buf_addr);
	PMD_DEBUG_RX("%s: Pushed mbuf %p, buffer %p into stack %d\n",
		     mpipe_name(priv), mbuf, buf_addr, priv->stack);
}

static inline void
mpipe_recv_fill_stack(struct mpipe_dev_priv *priv, int count)
{
	struct rte_mbuf *mbuf;
	int i;

	for (i = 0; i < count; i++) {
		mbuf = rte_mbuf_raw_alloc(priv->rx_mpool);
		if (!mbuf)
			break;
		mpipe_recv_push(priv, mbuf);
	}

	PMD_DEBUG_RX("%s: Filled %d/%d buffers\n", mpipe_name(priv), i, count);
}

static inline void
mpipe_recv_flush_stack(struct mpipe_dev_priv *priv)
{
	const int offset = priv->rx_offset & ~RTE_MEMPOOL_ALIGN_MASK;
	uint8_t in_port = priv->port_id;
	struct rte_mbuf *mbuf;
	void *va;

	while (1) {
		va = gxio_mpipe_pop_buffer(priv->context, priv->stack);
		if (!va)
			break;
		mbuf = RTE_PTR_SUB(va, offset);

		PMD_DEBUG_RX("%s: Flushing mbuf %p, va %p\n",
			     mpipe_name(priv), mbuf, va);

		mbuf->data_off    = (uintptr_t)va - (uintptr_t)mbuf->buf_addr;
		mbuf->refcnt      = 1;
		mbuf->nb_segs     = 1;
		mbuf->port        = in_port;
		mbuf->packet_type = 0;
		mbuf->data_len    = 0;
		mbuf->pkt_len     = 0;

		__rte_mbuf_raw_free(mbuf);
	}
}

static void
mpipe_register_segment(struct mpipe_dev_priv *priv, const struct rte_memseg *ms)
{
	size_t size = ms->hugepage_sz;
	uint8_t *addr, *end;
	int rc;

	for (addr = ms->addr, end = addr + ms->len; addr < end; addr += size) {
		rc = gxio_mpipe_register_page(priv->context, priv->stack, addr,
					      size, 0);
		if (rc < 0)
			break;
	}

	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Could not register memseg @%p, %d.\n",
			mpipe_name(priv), ms->addr, rc);
	} else {
		RTE_LOG(DEBUG, PMD, "%s: Registered segment %p - %p\n",
			mpipe_name(priv), ms->addr,
			RTE_PTR_ADD(ms->addr, ms->len - 1));
	}
}

static int
mpipe_recv_init(struct mpipe_dev_priv *priv)
{
	const struct rte_memseg *seg = rte_eal_get_physmem_layout();
	size_t stack_size;
	void *stack_mem;
	int rc;

	if (!priv->rx_mpool) {
		RTE_LOG(ERR, PMD, "%s: No buffer pool.\n",
			mpipe_name(priv));
		return -ENODEV;
	}

	/* Allocate one NotifRing for each queue. */
	rc = gxio_mpipe_alloc_notif_rings(priv->context, MPIPE_RX_MAX_QUEUES,
					  0, 0);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to allocate notif rings.\n",
			mpipe_name(priv));
		return rc;
	}
	priv->first_ring = rc;

	/* Allocate a NotifGroup. */
	rc = gxio_mpipe_alloc_notif_groups(priv->context, 1, 0, 0);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to allocate rx group.\n",
			mpipe_name(priv));
		return rc;
	}
	priv->notif_group = rc;

	/* Allocate required buckets. */
	rc = gxio_mpipe_alloc_buckets(priv->context, MPIPE_RX_BUCKETS, 0, 0);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to allocate buckets.\n",
			mpipe_name(priv));
		return rc;
	}
	priv->first_bucket = rc;

	rc = gxio_mpipe_alloc_buffer_stacks(priv->context, 1, 0, 0);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to allocate buffer stack.\n",
			mpipe_name(priv));
		return rc;
	}
	priv->stack = rc;

	while (seg && seg->addr)
		mpipe_register_segment(priv, seg++);

	stack_size = gxio_mpipe_calc_buffer_stack_bytes(MPIPE_RX_STACK_SIZE);
	stack_mem = rte_zmalloc(NULL, stack_size, 65536);
	if (!stack_mem) {
		RTE_LOG(ERR, PMD, "%s: Failed to allocate buffer memory.\n",
			mpipe_name(priv));
		return -ENOMEM;
	} else {
		RTE_LOG(DEBUG, PMD, "%s: Buffer stack memory %p - %p.\n",
			mpipe_name(priv), stack_mem,
			RTE_PTR_ADD(stack_mem, stack_size - 1));
	}

	rc = gxio_mpipe_init_buffer_stack(priv->context, priv->stack,
					  priv->rx_size_code, stack_mem,
					  stack_size, 0);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to initialize buffer stack.\n",
			mpipe_name(priv));
		return rc;
	}

	return 0;
}

static int
mpipe_xmit_init(struct mpipe_dev_priv *priv)
{
	size_t ring_size;
	void *ring_mem;
	int rc;

	/* Allocate eDMA ring. */
	rc = gxio_mpipe_alloc_edma_rings(priv->context, 1, 0, 0);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to alloc tx ring.\n",
			mpipe_name(priv));
		return rc;
	}
	priv->ering = rc;

	rc = mpipe_equeue_size(MPIPE_TX_DESCS);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Cannot allocate %d equeue descs.\n",
			mpipe_name(priv), (int)MPIPE_TX_DESCS);
		return -ENOMEM;
	}
	priv->equeue_size = rc;

	/* Initialize completion array. */
	ring_size = sizeof(priv->tx_comps[0]) * priv->equeue_size;
	priv->tx_comps = rte_zmalloc(NULL, ring_size, RTE_CACHE_LINE_SIZE);
	if (!priv->tx_comps) {
		RTE_LOG(ERR, PMD, "%s: Failed to allocate egress comps.\n",
			mpipe_name(priv));
		return -ENOMEM;
	}

	/* Allocate eDMA ring memory. */
	ring_size = sizeof(gxio_mpipe_edesc_t) * priv->equeue_size;
	ring_mem = rte_zmalloc(NULL, ring_size, ring_size);
	if (!ring_mem) {
		RTE_LOG(ERR, PMD, "%s: Failed to allocate egress descs.\n",
			mpipe_name(priv));
		return -ENOMEM;
	} else {
		RTE_LOG(DEBUG, PMD, "%s: eDMA ring memory %p - %p.\n",
			mpipe_name(priv), ring_mem,
			RTE_PTR_ADD(ring_mem, ring_size - 1));
	}

	/* Initialize eDMA ring. */
	rc = gxio_mpipe_equeue_init(&priv->equeue, priv->context, priv->ering,
				    priv->channel, ring_mem, ring_size, 0);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to init equeue\n",
			mpipe_name(priv));
		return rc;
	}

	return 0;
}

static int
mpipe_link_init(struct mpipe_dev_priv *priv)
{
	int rc;

	/* Open the link. */
	rc = gxio_mpipe_link_open(&priv->link, priv->context,
				  mpipe_name(priv), GXIO_MPIPE_LINK_AUTO_NONE);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to open link.\n",
			mpipe_name(priv));
		return rc;
	}

	/* Get the channel index. */
	rc = gxio_mpipe_link_channel(&priv->link);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Bad channel\n",
			mpipe_name(priv));
		return rc;
	}
	priv->channel = rc;

	return 0;
}

static int
mpipe_init(struct mpipe_dev_priv *priv)
{
	int rc;

	if (priv->initialized)
		return 0;

	rc = mpipe_recv_init(priv);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to init rx.\n",
			mpipe_name(priv));
		return rc;
	}

	rc = mpipe_xmit_init(priv);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to init tx.\n",
			mpipe_name(priv));
		rte_free(priv);
		return rc;
	}

	priv->initialized = 1;

	return 0;
}

static int
mpipe_start(struct rte_eth_dev *dev)
{
	struct mpipe_dev_priv *priv = mpipe_priv(dev);
	struct mpipe_channel_config config;
	struct mpipe_rx_queue *rx_queue;
	struct rte_eth_link eth_link;
	unsigned queue, buffers = 0;
	size_t ring_size;
	void *ring_mem;
	int rc;

	memset(&eth_link, 0, sizeof(eth_link));
	mpipe_dev_atomic_write_link_status(dev, &eth_link);

	rc = mpipe_init(priv);
	if (rc < 0)
		return rc;

	/* Initialize NotifRings. */
	for (queue = 0; queue < priv->nb_rx_queues; queue++) {
		rx_queue = mpipe_rx_queue(priv, queue);
		ring_size = rx_queue->q.nb_desc * sizeof(gxio_mpipe_idesc_t);

		ring_mem = rte_malloc(NULL, ring_size, ring_size);
		if (!ring_mem) {
			RTE_LOG(ERR, PMD, "%s: Failed to alloc rx descs.\n",
				mpipe_name(priv));
			return -ENOMEM;
		} else {
			RTE_LOG(DEBUG, PMD, "%s: iDMA ring %d memory %p - %p.\n",
				mpipe_name(priv), queue, ring_mem,
				RTE_PTR_ADD(ring_mem, ring_size - 1));
		}

		rc = gxio_mpipe_iqueue_init(&rx_queue->iqueue, priv->context,
					    priv->first_ring + queue, ring_mem,
					    ring_size, 0);
		if (rc < 0) {
			RTE_LOG(ERR, PMD, "%s: Failed to init rx queue.\n",
				mpipe_name(priv));
			return rc;
		}

		rx_queue->rx_ring_mem = ring_mem;
		buffers += rx_queue->q.nb_desc;
	}

	/* Initialize ingress NotifGroup and buckets. */
	rc = gxio_mpipe_init_notif_group_and_buckets(priv->context,
			priv->notif_group, priv->first_ring, priv->nb_rx_queues,
			priv->first_bucket, MPIPE_RX_BUCKETS,
			GXIO_MPIPE_BUCKET_STATIC_FLOW_AFFINITY);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to init group and buckets.\n",
			mpipe_name(priv));
		return rc;
	}

	/* Configure the classifier to deliver packets from this port. */
	config.enable = 1;
	config.first_bucket = priv->first_bucket;
	config.num_buckets = MPIPE_RX_BUCKETS;
	memset(&config.stacks, 0xff, sizeof(config.stacks));
	config.stacks.stacks[priv->rx_size_code] = priv->stack;
	config.head_room = priv->rx_offset & RTE_MEMPOOL_ALIGN_MASK;

	rc = mpipe_channel_config(priv->instance, priv->channel,
				  &config);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to setup classifier.\n",
			mpipe_name(priv));
		return rc;
	}

	/* Fill empty buffers into the buffer stack. */
	mpipe_recv_fill_stack(priv, buffers);

	/* Bring up the link. */
	mpipe_set_link_up(dev);

	/* Start xmit/recv on queues. */
	for (queue = 0; queue < priv->nb_tx_queues; queue++)
		mpipe_tx_queue(priv, queue)->q.link_status = ETH_LINK_UP;
	for (queue = 0; queue < priv->nb_rx_queues; queue++)
		mpipe_rx_queue(priv, queue)->q.link_status = ETH_LINK_UP;
	priv->running = 1;

	return 0;
}

static void
mpipe_stop(struct rte_eth_dev *dev)
{
	struct mpipe_dev_priv *priv = mpipe_priv(dev);
	struct mpipe_channel_config config;
	unsigned queue;
	int rc;

	for (queue = 0; queue < priv->nb_tx_queues; queue++)
		mpipe_tx_queue(priv, queue)->q.link_status = ETH_LINK_DOWN;
	for (queue = 0; queue < priv->nb_rx_queues; queue++)
		mpipe_rx_queue(priv, queue)->q.link_status = ETH_LINK_DOWN;

	/* Make sure the link_status writes land. */
	rte_wmb();

	/*
	 * Wait for link_status change to register with straggling datapath
	 * threads.
	 */
	mpipe_dp_wait(priv);

	/* Bring down the link. */
	mpipe_set_link_down(dev);

	/* Remove classifier rules. */
	memset(&config, 0, sizeof(config));
	rc = mpipe_channel_config(priv->instance, priv->channel,
				  &config);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to stop classifier.\n",
			mpipe_name(priv));
	}

	/* Flush completed xmit packets. */
	mpipe_xmit_flush(priv);

	/* Flush buffer stacks. */
	mpipe_recv_flush(priv);

	priv->running = 0;
}

static void
mpipe_close(struct rte_eth_dev *dev)
{
	struct mpipe_dev_priv *priv = mpipe_priv(dev);
	if (priv->running)
		mpipe_stop(dev);
}

static void
mpipe_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
{
	struct mpipe_dev_priv *priv = mpipe_priv(dev);
	struct mpipe_tx_queue *tx_queue;
	struct mpipe_rx_queue *rx_queue;
	unsigned i;
	uint16_t idx;

	memset(stats, 0, sizeof(*stats));

	for (i = 0; i < priv->nb_tx_queues; i++) {
		tx_queue = mpipe_tx_queue(priv, i);

		stats->opackets += tx_queue->q.stats.packets;
		stats->obytes   += tx_queue->q.stats.bytes;
		stats->oerrors  += tx_queue->q.stats.errors;

		idx = tx_queue->q.stat_idx;
		if (idx != (uint16_t)-1) {
			stats->q_opackets[idx] += tx_queue->q.stats.packets;
			stats->q_obytes[idx]   += tx_queue->q.stats.bytes;
			stats->q_errors[idx]   += tx_queue->q.stats.errors;
		}
	}

	for (i = 0; i < priv->nb_rx_queues; i++) {
		rx_queue = mpipe_rx_queue(priv, i);

		stats->ipackets  += rx_queue->q.stats.packets;
		stats->ibytes    += rx_queue->q.stats.bytes;
		stats->ierrors   += rx_queue->q.stats.errors;
		stats->rx_nombuf += rx_queue->q.stats.nomem;

		idx = rx_queue->q.stat_idx;
		if (idx != (uint16_t)-1) {
			stats->q_ipackets[idx] += rx_queue->q.stats.packets;
			stats->q_ibytes[idx]   += rx_queue->q.stats.bytes;
			stats->q_errors[idx]   += rx_queue->q.stats.errors;
		}
	}
}

static void
mpipe_stats_reset(struct rte_eth_dev *dev)
{
	struct mpipe_dev_priv *priv = mpipe_priv(dev);
	struct mpipe_tx_queue *tx_queue;
	struct mpipe_rx_queue *rx_queue;
	unsigned i;

	for (i = 0; i < priv->nb_tx_queues; i++) {
		tx_queue = mpipe_tx_queue(priv, i);
		memset(&tx_queue->q.stats, 0, sizeof(tx_queue->q.stats));
	}

	for (i = 0; i < priv->nb_rx_queues; i++) {
		rx_queue = mpipe_rx_queue(priv, i);
		memset(&rx_queue->q.stats, 0, sizeof(rx_queue->q.stats));
	}
}

static int
mpipe_queue_stats_mapping_set(struct rte_eth_dev *dev, uint16_t queue_id,
			      uint8_t stat_idx, uint8_t is_rx)
{
	struct mpipe_dev_priv *priv = mpipe_priv(dev);

	if (is_rx) {
		priv->rx_stat_mapping[stat_idx] = queue_id;
	} else {
		priv->tx_stat_mapping[stat_idx] = queue_id;
	}

	return 0;
}

static int
mpipe_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
		     uint16_t nb_desc, unsigned int socket_id __rte_unused,
		     const struct rte_eth_txconf *tx_conf __rte_unused)
{
	struct mpipe_tx_queue *tx_queue = dev->data->tx_queues[queue_idx];
	struct mpipe_dev_priv *priv = mpipe_priv(dev);
	uint16_t idx;

	tx_queue = rte_realloc(tx_queue, sizeof(*tx_queue),
			       RTE_CACHE_LINE_SIZE);
	if (!tx_queue) {
		RTE_LOG(ERR, PMD, "%s: Failed to allocate TX queue.\n",
			mpipe_name(priv));
		return -ENOMEM;
	}

	memset(&tx_queue->q, 0, sizeof(tx_queue->q));
	tx_queue->q.priv = priv;
	tx_queue->q.queue_idx = queue_idx;
	tx_queue->q.port_id = dev->data->port_id;
	tx_queue->q.nb_desc = nb_desc;

	tx_queue->q.stat_idx = -1;
	for (idx = 0; idx < RTE_ETHDEV_QUEUE_STAT_CNTRS; idx++) {
		if (priv->tx_stat_mapping[idx] == queue_idx)
			tx_queue->q.stat_idx = idx;
	}

	dev->data->tx_queues[queue_idx] = tx_queue;

	return 0;
}

static void
mpipe_tx_queue_release(void *_txq)
{
	rte_free(_txq);
}

static int
mpipe_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
		     uint16_t nb_desc, unsigned int socket_id __rte_unused,
		     const struct rte_eth_rxconf *rx_conf __rte_unused,
		     struct rte_mempool *mp)
{
	struct mpipe_rx_queue *rx_queue = dev->data->rx_queues[queue_idx];
	struct mpipe_dev_priv *priv = mpipe_priv(dev);
	uint16_t idx;
	int size, rc;

	rc = mpipe_iqueue_size(nb_desc);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Cannot allocate %d iqueue descs.\n",
			mpipe_name(priv), (int)nb_desc);
		return -ENOMEM;
	}

	if (rc != nb_desc) {
		RTE_LOG(WARNING, PMD, "%s: Extending RX descs from %d to %d.\n",
			mpipe_name(priv), (int)nb_desc, rc);
		nb_desc = rc;
	}

	size = sizeof(*rx_queue);
	rx_queue = rte_realloc(rx_queue, size, RTE_CACHE_LINE_SIZE);
	if (!rx_queue) {
		RTE_LOG(ERR, PMD, "%s: Failed to allocate RX queue.\n",
			mpipe_name(priv));
		return -ENOMEM;
	}

	memset(&rx_queue->q, 0, sizeof(rx_queue->q));
	rx_queue->q.priv = priv;
	rx_queue->q.nb_desc = nb_desc;
	rx_queue->q.port_id = dev->data->port_id;
	rx_queue->q.queue_idx = queue_idx;

	if (!priv->rx_mpool) {
		int size = (rte_pktmbuf_data_room_size(mp) -
			    RTE_PKTMBUF_HEADROOM -
			    MPIPE_RX_IP_ALIGN);

		priv->rx_offset = (sizeof(struct rte_mbuf) +
				   rte_pktmbuf_priv_size(mp) +
				   RTE_PKTMBUF_HEADROOM +
				   MPIPE_RX_IP_ALIGN);
		if (size < 0) {
			RTE_LOG(ERR, PMD, "%s: Bad buffer size %d.\n",
				mpipe_name(priv),
				rte_pktmbuf_data_room_size(mp));
			return -ENOMEM;
		}

		priv->rx_size_code = mpipe_buffer_size_index(size);
		priv->rx_mpool = mp;
	}

	if (priv->rx_mpool != mp) {
		RTE_LOG(WARNING, PMD, "%s: Ignoring multiple buffer pools.\n",
			mpipe_name(priv));
	}

	rx_queue->q.stat_idx = -1;
	for (idx = 0; idx < RTE_ETHDEV_QUEUE_STAT_CNTRS; idx++) {
		if (priv->rx_stat_mapping[idx] == queue_idx)
			rx_queue->q.stat_idx = idx;
	}

	dev->data->rx_queues[queue_idx] = rx_queue;

	return 0;
}

static void
mpipe_rx_queue_release(void *_rxq)
{
	rte_free(_rxq);
}

#define MPIPE_XGBE_ENA_HASH_MULTI	\
	(1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__ENA_HASH_MULTI_SHIFT)
#define MPIPE_XGBE_ENA_HASH_UNI		\
	(1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__ENA_HASH_UNI_SHIFT)
#define MPIPE_XGBE_COPY_ALL		\
	(1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__COPY_ALL_SHIFT)
#define MPIPE_GBE_ENA_MULTI_HASH	\
	(1UL << MPIPE_GBE_NETWORK_CONFIGURATION__MULTI_HASH_ENA_SHIFT)
#define MPIPE_GBE_ENA_UNI_HASH		\
	(1UL << MPIPE_GBE_NETWORK_CONFIGURATION__UNI_HASH_ENA_SHIFT)
#define MPIPE_GBE_COPY_ALL		\
	(1UL << MPIPE_GBE_NETWORK_CONFIGURATION__COPY_ALL_SHIFT)

static void
mpipe_promiscuous_enable(struct rte_eth_dev *dev)
{
	struct mpipe_dev_priv *priv = mpipe_priv(dev);
	int64_t reg;
	int addr;

	if (priv->is_xaui) {
		addr = MPIPE_XAUI_RECEIVE_CONFIGURATION;
		reg  = gxio_mpipe_link_mac_rd(&priv->link, addr);
		reg &= ~MPIPE_XGBE_ENA_HASH_MULTI;
		reg &= ~MPIPE_XGBE_ENA_HASH_UNI;
		reg |=  MPIPE_XGBE_COPY_ALL;
		gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
	} else {
		addr = MPIPE_GBE_NETWORK_CONFIGURATION;
		reg  = gxio_mpipe_link_mac_rd(&priv->link, addr);
		reg &= ~MPIPE_GBE_ENA_MULTI_HASH;
		reg &= ~MPIPE_GBE_ENA_UNI_HASH;
		reg |=  MPIPE_GBE_COPY_ALL;
		gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
	}
}

static void
mpipe_promiscuous_disable(struct rte_eth_dev *dev)
{
	struct mpipe_dev_priv *priv = mpipe_priv(dev);
	int64_t reg;
	int addr;

	if (priv->is_xaui) {
		addr = MPIPE_XAUI_RECEIVE_CONFIGURATION;
		reg  = gxio_mpipe_link_mac_rd(&priv->link, addr);
		reg |=  MPIPE_XGBE_ENA_HASH_MULTI;
		reg |=  MPIPE_XGBE_ENA_HASH_UNI;
		reg &= ~MPIPE_XGBE_COPY_ALL;
		gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
	} else {
		addr = MPIPE_GBE_NETWORK_CONFIGURATION;
		reg  = gxio_mpipe_link_mac_rd(&priv->link, addr);
		reg |=  MPIPE_GBE_ENA_MULTI_HASH;
		reg |=  MPIPE_GBE_ENA_UNI_HASH;
		reg &= ~MPIPE_GBE_COPY_ALL;
		gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
	}
}

static const struct eth_dev_ops mpipe_dev_ops = {
	.dev_infos_get	         = mpipe_infos_get,
	.dev_configure	         = mpipe_configure,
	.dev_start	         = mpipe_start,
	.dev_stop	         = mpipe_stop,
	.dev_close	         = mpipe_close,
	.stats_get	         = mpipe_stats_get,
	.stats_reset	         = mpipe_stats_reset,
	.queue_stats_mapping_set = mpipe_queue_stats_mapping_set,
	.tx_queue_setup	         = mpipe_tx_queue_setup,
	.rx_queue_setup	         = mpipe_rx_queue_setup,
	.tx_queue_release	 = mpipe_tx_queue_release,
	.rx_queue_release	 = mpipe_rx_queue_release,
	.link_update	         = mpipe_link_update,
	.dev_set_link_up         = mpipe_set_link_up,
	.dev_set_link_down       = mpipe_set_link_down,
	.promiscuous_enable      = mpipe_promiscuous_enable,
	.promiscuous_disable     = mpipe_promiscuous_disable,
};

static inline void
mpipe_xmit_null(struct mpipe_dev_priv *priv, int64_t start, int64_t end)
{
	gxio_mpipe_edesc_t null_desc = { { .bound = 1, .ns = 1 } };
	gxio_mpipe_equeue_t *equeue = &priv->equeue;
	int64_t slot;

	for (slot = start; slot < end; slot++) {
		gxio_mpipe_equeue_put_at(equeue, null_desc, slot);
	}
}

static void
mpipe_xmit_flush(struct mpipe_dev_priv *priv)
{
	gxio_mpipe_equeue_t *equeue = &priv->equeue;
	int64_t slot;

	/* Post a dummy descriptor and wait for its return. */
	slot = gxio_mpipe_equeue_reserve(equeue, 1);
	if (slot < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to reserve stop slot.\n",
			mpipe_name(priv));
		return;
	}

	mpipe_xmit_null(priv, slot, slot + 1);

	while (!gxio_mpipe_equeue_is_complete(equeue, slot, 1)) {
		rte_pause();
	}

	for (slot = 0; slot < priv->equeue_size; slot++) {
		if (priv->tx_comps[slot])
			rte_pktmbuf_free_seg(priv->tx_comps[slot]);
	}
}

static void
mpipe_recv_flush(struct mpipe_dev_priv *priv)
{
	uint8_t in_port = priv->port_id;
	struct mpipe_rx_queue *rx_queue;
	gxio_mpipe_iqueue_t *iqueue;
	gxio_mpipe_idesc_t idesc;
	struct rte_mbuf *mbuf;
	unsigned queue;

	/* Release packets on the buffer stack. */
	mpipe_recv_flush_stack(priv);

	/* Flush packets sitting in recv queues. */
	for (queue = 0; queue < priv->nb_rx_queues; queue++) {
		rx_queue = mpipe_rx_queue(priv, queue);
		iqueue = &rx_queue->iqueue;
		while (gxio_mpipe_iqueue_try_get(iqueue, &idesc) >= 0) {
			/* Skip idesc with the 'buffer error' bit set. */
			if (idesc.be)
				continue;
			mbuf = mpipe_recv_mbuf(priv, &idesc, in_port);
			rte_pktmbuf_free(mbuf);
		}
		rte_free(rx_queue->rx_ring_mem);
	}
}

static inline uint16_t
mpipe_do_xmit(struct mpipe_tx_queue *tx_queue, struct rte_mbuf **tx_pkts,
	      uint16_t nb_pkts)
{
	struct mpipe_dev_priv *priv = tx_queue->q.priv;
	gxio_mpipe_equeue_t *equeue = &priv->equeue;
	unsigned nb_bytes = 0;
	unsigned nb_sent = 0;
	int nb_slots, i;
	uint8_t port_id;

	PMD_DEBUG_TX("Trying to transmit %d packets on %s:%d.\n",
		     nb_pkts, mpipe_name(tx_queue->q.priv),
		     tx_queue->q.queue_idx);

	/* Optimistic assumption that we need exactly one slot per packet. */
	nb_slots = RTE_MIN(nb_pkts, MPIPE_TX_DESCS / 2);

	do {
		struct rte_mbuf *mbuf = NULL, *pkt = NULL;
		int64_t slot;

		/* Reserve eDMA ring slots. */
		slot = gxio_mpipe_equeue_try_reserve_fast(equeue, nb_slots);
		if (unlikely(slot < 0)) {
			break;
		}

		for (i = 0; i < nb_slots; i++) {
			unsigned idx = (slot + i) & (priv->equeue_size - 1);
			rte_prefetch0(priv->tx_comps[idx]);
		}

		/* Fill up slots with descriptor and completion info. */
		for (i = 0; i < nb_slots; i++) {
			unsigned idx = (slot + i) & (priv->equeue_size - 1);
			gxio_mpipe_edesc_t desc;
			struct rte_mbuf *next;

			/* Starting on a new packet? */
			if (likely(!mbuf)) {
				int room = nb_slots - i;

				pkt = mbuf = tx_pkts[nb_sent];

				/* Bail out if we run out of descs. */
				if (unlikely(pkt->nb_segs > room))
					break;

				nb_sent++;
			}

			/* We have a segment to send. */
			next = mbuf->next;

			if (priv->tx_comps[idx])
				rte_pktmbuf_free_seg(priv->tx_comps[idx]);

			port_id = (mbuf->port < RTE_MAX_ETHPORTS) ?
						mbuf->port : priv->port_id;
			desc = (gxio_mpipe_edesc_t) { {
				.va        = rte_pktmbuf_mtod(mbuf, uintptr_t),
				.xfer_size = rte_pktmbuf_data_len(mbuf),
				.bound     = next ? 0 : 1,
				.stack_idx = mpipe_mbuf_stack_index(priv, mbuf),
				.size      = priv->rx_size_code,
			} };
			if (mpipe_local.mbuf_push_debt[port_id] > 0) {
				mpipe_local.mbuf_push_debt[port_id]--;
				desc.hwb = 1;
				priv->tx_comps[idx] = NULL;
			} else
				priv->tx_comps[idx] = mbuf;

			nb_bytes += mbuf->data_len;
			gxio_mpipe_equeue_put_at(equeue, desc, slot + i);

			PMD_DEBUG_TX("%s:%d: Sending packet %p, len %d\n",
				     mpipe_name(priv),
				     tx_queue->q.queue_idx,
				     rte_pktmbuf_mtod(mbuf, void *),
				     rte_pktmbuf_data_len(mbuf));

			mbuf = next;
		}

		if (unlikely(nb_sent < nb_pkts)) {

			/* Fill remaining slots with null descriptors. */
			mpipe_xmit_null(priv, slot + i, slot + nb_slots);

			/*
			 * Calculate exact number of descriptors needed for
			 * the next go around.
			 */
			nb_slots = 0;
			for (i = nb_sent; i < nb_pkts; i++) {
				nb_slots += tx_pkts[i]->nb_segs;
			}

			nb_slots = RTE_MIN(nb_slots, MPIPE_TX_DESCS / 2);
		}
	} while (nb_sent < nb_pkts);

	tx_queue->q.stats.packets += nb_sent;
	tx_queue->q.stats.bytes   += nb_bytes;

	return nb_sent;
}

static inline uint16_t
mpipe_do_recv(struct mpipe_rx_queue *rx_queue, struct rte_mbuf **rx_pkts,
	      uint16_t nb_pkts)
{
	struct mpipe_dev_priv *priv = rx_queue->q.priv;
	gxio_mpipe_iqueue_t *iqueue = &rx_queue->iqueue;
	gxio_mpipe_idesc_t *first_idesc, *idesc, *last_idesc;
	uint8_t in_port = rx_queue->q.port_id;
	const unsigned look_ahead = 8;
	int room = nb_pkts, rc = 0;
	unsigned nb_packets = 0;
	unsigned nb_dropped = 0;
	unsigned nb_nomem = 0;
	unsigned nb_bytes = 0;
	unsigned nb_descs, i;

	while (room && !rc) {
		if (rx_queue->avail_descs < room) {
			rc = gxio_mpipe_iqueue_try_peek(iqueue,
							&rx_queue->next_desc);
			rx_queue->avail_descs = rc < 0 ? 0 : rc;
		}

		if (unlikely(!rx_queue->avail_descs)) {
			break;
		}

		nb_descs = RTE_MIN(room, rx_queue->avail_descs);

		first_idesc = rx_queue->next_desc;
		last_idesc  = first_idesc + nb_descs;

		rx_queue->next_desc   += nb_descs;
		rx_queue->avail_descs -= nb_descs;

		for (i = 1; i < look_ahead; i++) {
			rte_prefetch0(first_idesc + i);
		}

		PMD_DEBUG_RX("%s:%d: Trying to receive %d packets\n",
			     mpipe_name(rx_queue->q.priv),
			     rx_queue->q.queue_idx,
			     nb_descs);

		for (idesc = first_idesc; idesc < last_idesc; idesc++) {
			struct rte_mbuf *mbuf;

			PMD_DEBUG_RX("%s:%d: processing idesc %d/%d\n",
				     mpipe_name(priv),
				     rx_queue->q.queue_idx,
				     nb_packets, nb_descs);

			rte_prefetch0(idesc + look_ahead);

			PMD_DEBUG_RX("%s:%d: idesc %p, %s%s%s%s%s%s%s%s%s%s"
				     "size: %d, bkt: %d, chan: %d, ring: %d, sqn: %lu, va: %lu\n",
				     mpipe_name(priv),
				     rx_queue->q.queue_idx,
				     idesc,
				     idesc->me ? "me, " : "",
				     idesc->tr ? "tr, " : "",
				     idesc->ce ? "ce, " : "",
				     idesc->ct ? "ct, " : "",
				     idesc->cs ? "cs, " : "",
				     idesc->nr ? "nr, " : "",
				     idesc->sq ? "sq, " : "",
				     idesc->ts ? "ts, " : "",
				     idesc->ps ? "ps, " : "",
				     idesc->be ? "be, " : "",
				     idesc->l2_size,
				     idesc->bucket_id,
				     idesc->channel,
				     idesc->notif_ring,
				     (unsigned long)idesc->packet_sqn,
				     (unsigned long)idesc->va);

			if (unlikely(gxio_mpipe_idesc_has_error(idesc))) {
				nb_dropped++;
				gxio_mpipe_iqueue_drop(iqueue, idesc);
				PMD_DEBUG_RX("%s:%d: Descriptor error\n",
					     mpipe_name(rx_queue->q.priv),
					     rx_queue->q.queue_idx);
				continue;
			}

			if (mpipe_local.mbuf_push_debt[in_port] <
					MPIPE_BUF_DEBT_THRESHOLD)
				mpipe_local.mbuf_push_debt[in_port]++;
			else {
				mbuf = rte_mbuf_raw_alloc(priv->rx_mpool);
				if (unlikely(!mbuf)) {
					nb_nomem++;
					gxio_mpipe_iqueue_drop(iqueue, idesc);
					PMD_DEBUG_RX("%s:%d: alloc failure\n",
					     mpipe_name(rx_queue->q.priv),
					     rx_queue->q.queue_idx);
					continue;
				}

				mpipe_recv_push(priv, mbuf);
			}

			/* Get and setup the mbuf for the received packet. */
			mbuf = mpipe_recv_mbuf(priv, idesc, in_port);

			/* Update results and statistics counters. */
			rx_pkts[nb_packets] = mbuf;
			nb_bytes += mbuf->pkt_len;
			nb_packets++;
		}

		/*
		 * We release the ring in bursts, but do not track and release
		 * buckets.  This therefore breaks dynamic flow affinity, but
		 * we always operate in static affinity mode, and so we're OK
		 * with this optimization.
		 */
		gxio_mpipe_iqueue_advance(iqueue, nb_descs);
		gxio_mpipe_credit(iqueue->context, iqueue->ring, -1, nb_descs);

		/*
		 * Go around once more if we haven't yet peeked the queue, and
		 * if we have more room to receive.
		 */
		room = nb_pkts - nb_packets;
	}

	rx_queue->q.stats.packets += nb_packets;
	rx_queue->q.stats.bytes   += nb_bytes;
	rx_queue->q.stats.errors  += nb_dropped;
	rx_queue->q.stats.nomem   += nb_nomem;

	PMD_DEBUG_RX("%s:%d: RX: %d/%d pkts/bytes, %d/%d drops/nomem\n",
		     mpipe_name(rx_queue->q.priv), rx_queue->q.queue_idx,
		     nb_packets, nb_bytes, nb_dropped, nb_nomem);

	return nb_packets;
}

static uint16_t
mpipe_recv_pkts(void *_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
	struct mpipe_rx_queue *rx_queue = _rxq;
	uint16_t result = 0;

	if (rx_queue) {
		mpipe_dp_enter(rx_queue->q.priv);
		if (likely(rx_queue->q.link_status))
			result = mpipe_do_recv(rx_queue, rx_pkts, nb_pkts);
		mpipe_dp_exit(rx_queue->q.priv);
	}

	return result;
}

static uint16_t
mpipe_xmit_pkts(void *_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
	struct mpipe_tx_queue *tx_queue = _txq;
	uint16_t result = 0;

	if (tx_queue) {
		mpipe_dp_enter(tx_queue->q.priv);
		if (likely(tx_queue->q.link_status))
			result = mpipe_do_xmit(tx_queue, tx_pkts, nb_pkts);
		mpipe_dp_exit(tx_queue->q.priv);
	}

	return result;
}

static int
mpipe_link_mac(const char *ifname, uint8_t *mac)
{
	int rc, idx;
	char name[GXIO_MPIPE_LINK_NAME_LEN];

	for (idx = 0, rc = 0; !rc; idx++) {
		rc = gxio_mpipe_link_enumerate_mac(idx, name, mac);
		if (!rc && !strncmp(name, ifname, GXIO_MPIPE_LINK_NAME_LEN))
			return 0;
	}
	return -ENODEV;
}

static int
rte_pmd_mpipe_devinit(const char *ifname,
		      const char *params __rte_unused)
{
	gxio_mpipe_context_t *context;
	struct rte_eth_dev *eth_dev;
	struct mpipe_dev_priv *priv;
	int instance, rc;
	uint8_t *mac;

	/* Get the mPIPE instance that the device belongs to. */
	instance = gxio_mpipe_link_instance(ifname);
	context = mpipe_context(instance);
	if (!context) {
		RTE_LOG(ERR, PMD, "%s: No device for link.\n", ifname);
		return -ENODEV;
	}

	priv = rte_zmalloc(NULL, sizeof(*priv), 0);
	if (!priv) {
		RTE_LOG(ERR, PMD, "%s: Failed to allocate priv.\n", ifname);
		return -ENOMEM;
	}

	memset(&priv->tx_stat_mapping, 0xff, sizeof(priv->tx_stat_mapping));
	memset(&priv->rx_stat_mapping, 0xff, sizeof(priv->rx_stat_mapping));
	priv->context = context;
	priv->instance = instance;
	priv->is_xaui = (strncmp(ifname, "xgbe", 4) == 0);
	priv->channel = -1;

	mac = priv->mac_addr.addr_bytes;
	rc = mpipe_link_mac(ifname, mac);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to enumerate link.\n", ifname);
		rte_free(priv);
		return -ENODEV;
	}

	eth_dev = rte_eth_dev_allocate(ifname, RTE_ETH_DEV_VIRTUAL);
	if (!eth_dev) {
		RTE_LOG(ERR, PMD, "%s: Failed to allocate device.\n", ifname);
		rte_free(priv);
		return -ENOMEM;
	}

	RTE_LOG(INFO, PMD, "%s: Initialized mpipe device"
		"(mac %02x:%02x:%02x:%02x:%02x:%02x).\n",
		ifname, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);

	priv->eth_dev = eth_dev;
	priv->port_id = eth_dev->data->port_id;
	eth_dev->data->dev_private = priv;
	eth_dev->data->mac_addrs = &priv->mac_addr;

	eth_dev->data->dev_flags = 0;
	eth_dev->data->kdrv = RTE_KDRV_NONE;
	eth_dev->driver = NULL;
	eth_dev->data->drv_name = drivername;
	eth_dev->data->numa_node = instance;

	eth_dev->dev_ops      = &mpipe_dev_ops;
	eth_dev->rx_pkt_burst = &mpipe_recv_pkts;
	eth_dev->tx_pkt_burst = &mpipe_xmit_pkts;

	rc = mpipe_link_init(priv);
	if (rc < 0) {
		RTE_LOG(ERR, PMD, "%s: Failed to init link.\n",
			mpipe_name(priv));
		return rc;
	}

	return 0;
}

static struct rte_driver pmd_mpipe_xgbe_drv = {
	.type = PMD_VDEV,
	.init = rte_pmd_mpipe_devinit,
};

static struct rte_driver pmd_mpipe_gbe_drv = {
	.type = PMD_VDEV,
	.init = rte_pmd_mpipe_devinit,
};

PMD_REGISTER_DRIVER(pmd_mpipe_xgbe_drv, net_mpipe_xgbe);
PMD_REGISTER_DRIVER(pmd_mpipe_gbe_drv, net_mpipe_gbe);

static void __attribute__((constructor, used))
mpipe_init_contexts(void)
{
	struct mpipe_context *context;
	int rc, instance;

	for (instance = 0; instance < GXIO_MPIPE_INSTANCE_MAX; instance++) {
		context = &mpipe_contexts[instance];

		rte_spinlock_init(&context->lock);
		rc = gxio_mpipe_init(&context->context, instance);
		if (rc < 0)
			break;
	}

	mpipe_instances = instance;
}