numam-dpdk/drivers/raw/ifpga/afu_pmd_n3000.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2022 Intel Corporation
 */

#include <errno.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <inttypes.h>
#include <unistd.h>
#include <fcntl.h>
#include <poll.h>
#include <sys/eventfd.h>
#include <sys/ioctl.h>

#include <rte_eal.h>
#include <rte_malloc.h>
#include <rte_memcpy.h>
#include <rte_io.h>
#include <rte_vfio.h>
#include <rte_bus_pci.h>
#include <rte_bus_ifpga.h>
#include <rte_rawdev.h>

#include "afu_pmd_core.h"
#include "afu_pmd_n3000.h"

static int nlb_afu_config(struct afu_rawdev *dev)
{
	struct n3000_afu_priv *priv = NULL;
	struct rte_pmd_afu_nlb_cfg *cfg = NULL;
	struct nlb_csr_cfg v;

	if (!dev)
		return -EINVAL;

	if (!dev->priv)
		return -ENOENT;

	priv = (struct n3000_afu_priv *)dev->priv;
	cfg = &priv->nlb_cfg;

	v.csr = 0;

	if (cfg->cont)
		v.cont = 1;

	if (cfg->cache_policy == NLB_WRPUSH_I)
		v.wrpush_i = 1;
	else
		v.wrthru_en = cfg->cache_policy;

	if (cfg->cache_hint == NLB_RDLINE_MIXED)
		v.rdsel = 3;
	else
		v.rdsel = cfg->cache_hint;

	v.mode = cfg->mode;
	v.chsel = cfg->read_vc;
	v.wr_chsel = cfg->write_vc;
	v.wrfence_chsel = cfg->wrfence_vc;
	v.wrthru_en = cfg->cache_policy;
	v.multicl_len = cfg->multi_cl - 1;

	IFPGA_RAWDEV_PMD_DEBUG("cfg: 0x%08x", v.csr);
	rte_write32(v.csr, priv->nlb_ctx.addr + CSR_CFG);

	return 0;
}

static void nlb_afu_report(struct afu_rawdev *dev, uint32_t cl)
{
	struct n3000_afu_priv *priv = NULL;
	struct rte_pmd_afu_nlb_cfg *cfg = NULL;
	struct nlb_dsm_status *stat = NULL;
	uint64_t ticks = 0;
	double num, rd_bw, wr_bw;

	if (!dev || !dev->priv)
		return;

	priv = (struct n3000_afu_priv *)dev->priv;

	cfg = &priv->nlb_cfg;
	stat = priv->nlb_ctx.status_ptr;

	if (cfg->cont)
		ticks = stat->num_clocks - stat->start_overhead;
	else
		ticks = stat->num_clocks -
			(stat->start_overhead + stat->end_overhead);

	if (cfg->freq_mhz == 0)
		cfg->freq_mhz = 200;

	num = (double)stat->num_reads;
	rd_bw = (num * CLS_TO_SIZE(1) * MHZ(cfg->freq_mhz)) / ticks;
	num = (double)stat->num_writes;
	wr_bw = (num * CLS_TO_SIZE(1) * MHZ(cfg->freq_mhz)) / ticks;

	printf("Cachelines  Read_Count Write_Count Clocks@%uMHz   "
		"Rd_Bandwidth   Wr_Bandwidth\n", cfg->freq_mhz);
	printf("%10u  %10u %11u  %12"PRIu64"   %7.3f GB/s   %7.3f GB/s\n",
		cl, stat->num_reads, stat->num_writes, ticks,
		rd_bw / 1e9, wr_bw / 1e9);
}

static int nlb_afu_test(struct afu_rawdev *dev)
{
	struct n3000_afu_priv *priv = NULL;
	struct nlb_afu_ctx *ctx = NULL;
	struct rte_pmd_afu_nlb_cfg *cfg = NULL;
	struct nlb_csr_ctl ctl;
	uint32_t *ptr = NULL;
	uint32_t i, j, cl, val = 0;
	uint64_t sval = 0;
	int ret = 0;

	if (!dev)
		return -EINVAL;

	if (!dev->priv)
		return -ENOENT;

	priv = (struct n3000_afu_priv *)dev->priv;
	ctx = &priv->nlb_ctx;
	cfg = &priv->nlb_cfg;

	/* initialize registers */
	IFPGA_RAWDEV_PMD_DEBUG("dsm_addr: 0x%"PRIx64, ctx->dsm_iova);
	rte_write64(ctx->dsm_iova, ctx->addr + CSR_AFU_DSM_BASEL);

	ctl.csr = 0;
	rte_write32(ctl.csr, ctx->addr + CSR_CTL);
	ctl.reset = 1;
	rte_write32(ctl.csr, ctx->addr + CSR_CTL);

	IFPGA_RAWDEV_PMD_DEBUG("src_addr: 0x%"PRIx64, ctx->src_iova);
	rte_write64(SIZE_TO_CLS(ctx->src_iova), ctx->addr + CSR_SRC_ADDR);
	IFPGA_RAWDEV_PMD_DEBUG("dst_addr: 0x%"PRIx64, ctx->dest_iova);
	rte_write64(SIZE_TO_CLS(ctx->dest_iova), ctx->addr + CSR_DST_ADDR);

	ret = nlb_afu_config(dev);
	if (ret)
		return ret;

	/* initialize src data */
	ptr = (uint32_t *)ctx->src_ptr;
	j = CLS_TO_SIZE(cfg->end) >> 2;
	for (i = 0; i < j; i++)
		*ptr++ = i;

	/* start test */
	for (cl = cfg->begin; cl <= cfg->end; cl += cfg->multi_cl) {
		memset(ctx->dest_ptr, 0, CLS_TO_SIZE(cl));
		memset(ctx->dsm_ptr, 0, DSM_SIZE);

		ctl.csr = 0;
		rte_write32(ctl.csr, ctx->addr + CSR_CTL);
		ctl.reset = 1;
		rte_write32(ctl.csr, ctx->addr + CSR_CTL);

		rte_write32(cl, ctx->addr + CSR_NUM_LINES);

		rte_delay_us(10);

		ctl.start = 1;
		rte_write32(ctl.csr, ctx->addr + CSR_CTL);

		if (cfg->cont) {
			rte_delay_ms(cfg->timeout * 1000);
			ctl.force_completion = 1;
			rte_write32(ctl.csr, ctx->addr + CSR_CTL);
			ret = dsm_poll_timeout(&ctx->status_ptr->test_complete,
				val, (val & 0x1) == 1, DSM_POLL_INTERVAL,
				DSM_TIMEOUT);
			if (ret) {
				printf("DSM poll timeout\n");
				goto end;
			}
		} else {
			ret = dsm_poll_timeout(&ctx->status_ptr->test_complete,
				val, (val & 0x1) == 1, DSM_POLL_INTERVAL,
				DSM_TIMEOUT);
			if (ret) {
				printf("DSM poll timeout\n");
				goto end;
			}
			ctl.force_completion = 1;
			rte_write32(ctl.csr, ctx->addr + CSR_CTL);
		}

		nlb_afu_report(dev, cl);

		i = 0;
		while (i++ < 100) {
			sval = rte_read64(ctx->addr + CSR_STATUS1);
			if (sval == 0)
				break;
			rte_delay_us(1000);
		}

		ptr = (uint32_t *)ctx->dest_ptr;
		j = CLS_TO_SIZE(cl) >> 2;
		for (i = 0; i < j; i++) {
			if (*ptr++ != i) {
				IFPGA_RAWDEV_PMD_ERR("Data mismatch @ %u", i);
				break;
			}
		}
	}

end:
	return ret;
}

static void dma_afu_buf_free(struct dma_afu_ctx *ctx)
{
	int i = 0;

	if (!ctx)
		return;

	for (i = 0; i < NUM_DMA_BUF; i++) {
		rte_free(ctx->dma_buf[i]);
		ctx->dma_buf[i] = NULL;
	}

	rte_free(ctx->data_buf);
	ctx->data_buf = NULL;

	rte_free(ctx->ref_buf);
	ctx->ref_buf = NULL;
}

static int dma_afu_buf_alloc(struct dma_afu_ctx *ctx,
	struct rte_pmd_afu_dma_cfg *cfg)
{
	size_t page_sz = sysconf(_SC_PAGE_SIZE);
	int i, ret = 0;

	if (!ctx || !cfg)
		return -EINVAL;

	for (i = 0; i < NUM_DMA_BUF; i++) {
		ctx->dma_buf[i] = (uint64_t *)rte_zmalloc(NULL, cfg->size,
			TEST_MEM_ALIGN);
		if (!ctx->dma_buf[i]) {
			ret = -ENOMEM;
			goto free_dma_buf;
		}
		ctx->dma_iova[i] = rte_malloc_virt2iova(ctx->dma_buf[i]);
		if (ctx->dma_iova[i] == RTE_BAD_IOVA) {
			ret = -ENOMEM;
			goto free_dma_buf;
		}
	}

	ctx->data_buf = rte_malloc(NULL, cfg->length, page_sz);
	if (!ctx->data_buf) {
		ret = -ENOMEM;
		goto free_dma_buf;
	}

	ctx->ref_buf = rte_malloc(NULL, cfg->length, page_sz);
	if (!ctx->ref_buf) {
		ret = -ENOMEM;
		goto free_data_buf;
	}

	return 0;

free_data_buf:
	rte_free(ctx->data_buf);
	ctx->data_buf = NULL;
free_dma_buf:
	for (i = 0; i < NUM_DMA_BUF; i++) {
		rte_free(ctx->dma_buf[i]);
		ctx->dma_buf[i] = NULL;
	}
	return ret;
}

static void dma_afu_buf_init(struct dma_afu_ctx *ctx, size_t size)
{
	int *ptr = NULL;
	size_t i = 0;
	size_t dword_size = 0;

	if (!ctx || !size)
		return;

	ptr = (int *)ctx->ref_buf;

	if (ctx->pattern) {
		memset(ptr, ctx->pattern, size);
	} else {
		srand(99);
		dword_size = size >> 2;
		for (i = 0; i < dword_size; i++)
			*ptr++ = rand();
	}
	rte_memcpy(ctx->data_buf, ctx->ref_buf, size);
}

static int dma_afu_buf_verify(struct dma_afu_ctx *ctx, size_t size)
{
	uint8_t *src = NULL;
	uint8_t *dst = NULL;
	size_t i = 0;
	int n = 0;

	if (!ctx || !size)
		return -EINVAL;

	src = (uint8_t *)ctx->ref_buf;
	dst = (uint8_t *)ctx->data_buf;

	if (memcmp(src, dst, size)) {
		printf("Transfer is corrupted\n");
		if (ctx->verbose) {
			for (i = 0; i < size; i++) {
				if (*src != *dst) {
					if (++n >= ERR_CHECK_LIMIT)
						break;
					printf("Mismatch at 0x%zx, "
						"Expected %02x  Actual %02x\n",
						i, *src, *dst);
				}
				src++;
				dst++;
			}
			if (n < ERR_CHECK_LIMIT) {
				printf("Found %d error bytes\n", n);
			} else {
				printf("......\n");
				printf("Found more than %d error bytes\n", n);
			}
		}
		return -1;
	}

	printf("Transfer is verified\n");
	return 0;
}

static void blk_write64(uint64_t *dev_addr, uint64_t *host_addr, uint64_t bytes)
{
	uint64_t qwords = bytes / sizeof(uint64_t);

	if (!IS_ALIGNED_QWORD((uint64_t)dev_addr) ||
		!IS_ALIGNED_QWORD((uint64_t)bytes))
		return;

	for (; qwords > 0; qwords--, host_addr++, dev_addr++)
		rte_write64(*host_addr, dev_addr);
}

static void blk_read64(uint64_t *dev_addr, uint64_t *host_addr, uint64_t bytes)
{
	uint64_t qwords = bytes / sizeof(uint64_t);

	if (!IS_ALIGNED_QWORD((uint64_t)dev_addr) ||
		!IS_ALIGNED_QWORD((uint64_t)bytes))
		return;

	for (; qwords > 0; qwords--, host_addr++, dev_addr++)
		*host_addr = rte_read64(dev_addr);
}

static void switch_ase_page(struct dma_afu_ctx *ctx, uint64_t addr)
{
	uint64_t requested_page = addr & ~DMA_ASE_WINDOW_MASK;

	if (!ctx)
		return;

	if (requested_page != ctx->cur_ase_page) {
		rte_write64(requested_page, ctx->ase_ctrl_addr);
		ctx->cur_ase_page = requested_page;
	}
}

static int ase_write_unaligned(struct dma_afu_ctx *ctx, uint64_t dev_addr,
	uint64_t host_addr, uint32_t count)
{
	uint64_t dev_aligned_addr = 0;
	uint64_t shift = 0;
	uint64_t val = 0;
	uintptr_t addr = (uintptr_t)host_addr;  /* transfer to pointer size */

	IFPGA_RAWDEV_PMD_DEBUG("0x%"PRIx64" --> 0x%"PRIx64" (0x%x)", host_addr,
		dev_addr, count);

	if (!ctx || (count >= QWORD_BYTES))
		return -EINVAL;

	if (!count)
		return 0;

	switch_ase_page(ctx, dev_addr);

	shift = dev_addr % QWORD_BYTES;
	dev_aligned_addr = (dev_addr - shift) & DMA_ASE_WINDOW_MASK;
	val = rte_read64(ctx->ase_data_addr + dev_aligned_addr);
	rte_memcpy(((char *)(&val)) + shift, (void *)addr, count);

	/* write back to device */
	rte_write64(val, ctx->ase_data_addr + dev_aligned_addr);

	return 0;
}

static int ase_write(struct dma_afu_ctx *ctx, uint64_t *dst_ptr,
	uint64_t *src_ptr, uint64_t *count)
{
	uint64_t src = *src_ptr;
	uint64_t dst = *dst_ptr;
	uint64_t align_bytes = *count;
	uint64_t offset = 0;
	uint64_t left_in_page = DMA_ASE_WINDOW;
	uint64_t size_to_copy = 0;

	IFPGA_RAWDEV_PMD_DEBUG("0x%"PRIx64" --> 0x%"PRIx64" (0x%"PRIx64")", src, dst,
		align_bytes);

	if (!ctx || !IS_ALIGNED_DWORD(dst))
		return -EINVAL;

	if (align_bytes < DWORD_BYTES)
		return 0;

	if (!IS_ALIGNED_QWORD(dst)) {
		/* Write out a single DWORD to get QWORD aligned */
		switch_ase_page(ctx, dst);
		offset = dst & DMA_ASE_WINDOW_MASK;

		rte_write32(*(uint32_t *)(uintptr_t)src,
			ctx->ase_data_addr + offset);
		src += DWORD_BYTES;
		dst += DWORD_BYTES;
		align_bytes -= DWORD_BYTES;
	}

	if (!align_bytes)
		return 0;

	/* Write out blocks of 64-bit values */
	while (align_bytes >= QWORD_BYTES) {
		left_in_page -= dst & DMA_ASE_WINDOW_MASK;
		size_to_copy =
			MIN(left_in_page, (align_bytes & ~(QWORD_BYTES - 1)));
		if (size_to_copy < QWORD_BYTES)
			break;
		switch_ase_page(ctx, dst);
		offset = dst & DMA_ASE_WINDOW_MASK;
		blk_write64((uint64_t *)(ctx->ase_data_addr + offset),
			(uint64_t *)(uintptr_t)src, size_to_copy);
		src += size_to_copy;
		dst += size_to_copy;
		align_bytes -= size_to_copy;
	}

	if (align_bytes >= DWORD_BYTES) {
		/* Write out remaining DWORD */
		switch_ase_page(ctx, dst);
		offset = dst & DMA_ASE_WINDOW_MASK;
		rte_write32(*(uint32_t *)(uintptr_t)src,
			ctx->ase_data_addr + offset);
		src += DWORD_BYTES;
		dst += DWORD_BYTES;
		align_bytes -= DWORD_BYTES;
	}

	*src_ptr = src;
	*dst_ptr = dst;
	*count = align_bytes;

	return 0;
}

static int ase_host_to_fpga(struct dma_afu_ctx *ctx, uint64_t *dst_ptr,
	uint64_t *src_ptr, uint64_t count)
{
	uint64_t dst = *dst_ptr;
	uint64_t src = *src_ptr;
	uint64_t count_left = count;
	uint64_t unaligned_size = 0;
	int ret = 0;

	IFPGA_RAWDEV_PMD_DEBUG("0x%"PRIx64" --> 0x%"PRIx64" (0x%"PRIx64")", src, dst,
		count);

	/* aligns address to 8 byte using dst masking method */
	if (!IS_ALIGNED_DWORD(dst) && !IS_ALIGNED_QWORD(dst)) {
		unaligned_size = QWORD_BYTES - (dst % QWORD_BYTES);
		if (unaligned_size > count_left)
			unaligned_size = count_left;
		ret = ase_write_unaligned(ctx, dst, src, unaligned_size);
		if (ret)
			return ret;
		count_left -= unaligned_size;
		src += unaligned_size;
		dst += unaligned_size;
	}

	/* Handles 8/4 byte MMIO transfer */
	ret = ase_write(ctx, &dst, &src, &count_left);
	if (ret)
		return ret;

	/* Left over unaligned bytes transferred using dst masking method */
	unaligned_size = QWORD_BYTES - (dst % QWORD_BYTES);
	if (unaligned_size > count_left)
		unaligned_size = count_left;

	ret = ase_write_unaligned(ctx, dst, src, unaligned_size);
	if (ret)
		return ret;

	count_left -= unaligned_size;
	*dst_ptr = dst + unaligned_size;
	*src_ptr = src + unaligned_size;

	return 0;
}

static int ase_read_unaligned(struct dma_afu_ctx *ctx, uint64_t dev_addr,
	uint64_t host_addr, uint32_t count)
{
	uint64_t dev_aligned_addr = 0;
	uint64_t shift = 0;
	uint64_t val = 0;
	uintptr_t addr = (uintptr_t)host_addr;  /* transfer to pointer size */

	IFPGA_RAWDEV_PMD_DEBUG("0x%"PRIx64" <-- 0x%"PRIx64" (0x%x)", host_addr,
		dev_addr, count);

	if (!ctx || (count >= QWORD_BYTES))
		return -EINVAL;

	if (!count)
		return 0;

	switch_ase_page(ctx, dev_addr);

	shift = dev_addr % QWORD_BYTES;
	dev_aligned_addr = (dev_addr - shift) & DMA_ASE_WINDOW_MASK;
	val = rte_read64(ctx->ase_data_addr + dev_aligned_addr);
	rte_memcpy((void *)addr, ((char *)(&val)) + shift, count);

	return 0;
}

static int ase_read(struct dma_afu_ctx *ctx, uint64_t *src_ptr,
	uint64_t *dst_ptr, uint64_t *count)
{
	uint64_t src = *src_ptr;
	uint64_t dst = *dst_ptr;
	uint64_t align_bytes = *count;
	uint64_t offset = 0;
	uint64_t left_in_page = DMA_ASE_WINDOW;
	uint64_t size_to_copy = 0;

	IFPGA_RAWDEV_PMD_DEBUG("0x%"PRIx64" <-- 0x%"PRIx64" (0x%"PRIx64")", dst, src,
		align_bytes);

	if (!ctx || !IS_ALIGNED_DWORD(src))
		return -EINVAL;

	if (align_bytes < DWORD_BYTES)
		return 0;

	if (!IS_ALIGNED_QWORD(src)) {
		/* Read a single DWORD to get QWORD aligned */
		switch_ase_page(ctx, src);
		offset = src & DMA_ASE_WINDOW_MASK;
		*(uint32_t *)(uintptr_t)dst =
			rte_read32(ctx->ase_data_addr + offset);
		src += DWORD_BYTES;
		dst += DWORD_BYTES;
		align_bytes -= DWORD_BYTES;
	}

	if (!align_bytes)
		return 0;

	/* Read blocks of 64-bit values */
	while (align_bytes >= QWORD_BYTES) {
		left_in_page -= src & DMA_ASE_WINDOW_MASK;
		size_to_copy =
			MIN(left_in_page, (align_bytes & ~(QWORD_BYTES - 1)));
		if (size_to_copy < QWORD_BYTES)
			break;
		switch_ase_page(ctx, src);
		offset = src & DMA_ASE_WINDOW_MASK;
		blk_read64((uint64_t *)(ctx->ase_data_addr + offset),
			(uint64_t *)(uintptr_t)dst, size_to_copy);
		src += size_to_copy;
		dst += size_to_copy;
		align_bytes -= size_to_copy;
	}

	if (align_bytes >= DWORD_BYTES) {
		/* Read remaining DWORD */
		switch_ase_page(ctx, src);
		offset = src & DMA_ASE_WINDOW_MASK;
		*(uint32_t *)(uintptr_t)dst =
			rte_read32(ctx->ase_data_addr + offset);
		src += DWORD_BYTES;
		dst += DWORD_BYTES;
		align_bytes -= DWORD_BYTES;
	}

	*src_ptr = src;
	*dst_ptr = dst;
	*count = align_bytes;

	return 0;
}

static int ase_fpga_to_host(struct dma_afu_ctx *ctx, uint64_t *src_ptr,
	uint64_t *dst_ptr, uint64_t count)
{
	uint64_t src = *src_ptr;
	uint64_t dst = *dst_ptr;
	uint64_t count_left = count;
	uint64_t unaligned_size = 0;
	int ret = 0;

	IFPGA_RAWDEV_PMD_DEBUG("0x%"PRIx64" --> 0x%"PRIx64" (0x%"PRIx64")", src, dst,
		count);

	/* Aligns address to 8 byte using src masking method */
	if (!IS_ALIGNED_DWORD(src) && !IS_ALIGNED_QWORD(src)) {
		unaligned_size = QWORD_BYTES - (src % QWORD_BYTES);
		if (unaligned_size > count_left)
			unaligned_size = count_left;
		ret = ase_read_unaligned(ctx, src, dst, unaligned_size);
		if (ret)
			return ret;
		count_left -= unaligned_size;
		dst += unaligned_size;
		src += unaligned_size;
	}

	/* Handles 8/4 byte MMIO transfer */
	ret = ase_read(ctx, &src, &dst, &count_left);
	if (ret)
		return ret;

	/* Left over unaligned bytes transferred using src masking method */
	unaligned_size = QWORD_BYTES - (src % QWORD_BYTES);
	if (unaligned_size > count_left)
		unaligned_size = count_left;

	ret = ase_read_unaligned(ctx, src, dst, unaligned_size);
	if (ret)
		return ret;

	count_left -= unaligned_size;
	*dst_ptr = dst + unaligned_size;
	*src_ptr = src + unaligned_size;

	return 0;
}

static void clear_interrupt(struct dma_afu_ctx *ctx)
{
	/* clear interrupt by writing 1 to IRQ bit in status register */
	msgdma_status status;

	if (!ctx)
		return;

	status.csr = 0;
	status.irq = 1;
	rte_write32(status.csr, CSR_STATUS(ctx->csr_addr));
}

static int poll_interrupt(struct dma_afu_ctx *ctx)
{
	struct pollfd pfd = {0};
	uint64_t count = 0;
	ssize_t bytes_read = 0;
	int poll_ret = 0;
	int ret = 0;

	if (!ctx || (ctx->event_fd < 0))
		return -EINVAL;

	pfd.fd = ctx->event_fd;
	pfd.events = POLLIN;
	poll_ret = poll(&pfd, 1, DMA_TIMEOUT_MSEC);
	if (poll_ret < 0) {
		IFPGA_RAWDEV_PMD_ERR("Error %s", strerror(errno));
		ret = -EFAULT;
		goto out;
	} else if (poll_ret == 0) {
		IFPGA_RAWDEV_PMD_ERR("Timeout");
		ret = -ETIMEDOUT;
	} else {
		bytes_read = read(pfd.fd, &count, sizeof(count));
		if (bytes_read > 0) {
			if (ctx->verbose)
				IFPGA_RAWDEV_PMD_DEBUG("Successful, ret %d, cnt %"PRIu64,
					poll_ret, count);
			ret = 0;
		} else {
			IFPGA_RAWDEV_PMD_ERR("Failed %s", bytes_read > 0 ?
				strerror(errno) : "zero bytes read");
			ret = -EIO;
		}
	}
out:
	clear_interrupt(ctx);
	return ret;
}

static void send_descriptor(struct dma_afu_ctx *ctx, msgdma_ext_desc *desc)
{
	msgdma_status status;
	uint64_t fpga_queue_full = 0;

	if (!ctx)
		return;

	if (ctx->verbose) {
		IFPGA_RAWDEV_PMD_DEBUG("descriptor.rd_address = 0x%x%08x",
			desc->rd_address_ext, desc->rd_address);
		IFPGA_RAWDEV_PMD_DEBUG("descriptor.wr_address = 0x%x%08x",
			desc->wr_address_ext, desc->wr_address);
		IFPGA_RAWDEV_PMD_DEBUG("descriptor.len = %u", desc->len);
		IFPGA_RAWDEV_PMD_DEBUG("descriptor.wr_burst_count = %u",
			desc->wr_burst_count);
		IFPGA_RAWDEV_PMD_DEBUG("descriptor.rd_burst_count = %u",
			desc->rd_burst_count);
		IFPGA_RAWDEV_PMD_DEBUG("descriptor.wr_stride %u", desc->wr_stride);
		IFPGA_RAWDEV_PMD_DEBUG("descriptor.rd_stride %u", desc->rd_stride);
	}

	do {
		status.csr = rte_read32(CSR_STATUS(ctx->csr_addr));
		if (fpga_queue_full++ > 100000000) {
			IFPGA_RAWDEV_PMD_DEBUG("DMA queue full retry");
			fpga_queue_full = 0;
		}
	} while (status.desc_buf_full);

	blk_write64((uint64_t *)ctx->desc_addr, (uint64_t *)desc,
		sizeof(*desc));
}

static int do_dma(struct dma_afu_ctx *ctx, uint64_t dst, uint64_t src,
	int count, int is_last_desc, fpga_dma_type type, int intr_en)
{
	msgdma_ext_desc *desc = NULL;
	int alignment_offset = 0;
	int segment_size = 0;

	if (!ctx)
		return -EINVAL;

	/* src, dst and count must be 64-byte aligned */
	if (!IS_DMA_ALIGNED(src) || !IS_DMA_ALIGNED(dst) ||
		!IS_DMA_ALIGNED(count))
		return -EINVAL;
	memset(ctx->desc_buf, 0, sizeof(msgdma_ext_desc));

	/* these fields are fixed for all DMA transfers */
	desc = ctx->desc_buf;
	desc->seq_num = 0;
	desc->wr_stride = 1;
	desc->rd_stride = 1;
	desc->control.go = 1;
	if (intr_en)
		desc->control.transfer_irq_en = 1;
	else
		desc->control.transfer_irq_en = 0;

	if (!is_last_desc)
		desc->control.early_done_en = 1;
	else
		desc->control.early_done_en = 0;

	if (type == FPGA_TO_FPGA) {
		desc->rd_address = src & DMA_MASK_32_BIT;
		desc->wr_address = dst & DMA_MASK_32_BIT;
		desc->len = count;
		desc->wr_burst_count = 4;
		desc->rd_burst_count = 4;
		desc->rd_address_ext = (src >> 32) & DMA_MASK_32_BIT;
		desc->wr_address_ext = (dst >> 32) & DMA_MASK_32_BIT;
		send_descriptor(ctx, desc);
	} else {
		/* check CCIP (host) address is aligned to 4CL (256B) */
		alignment_offset = (type == HOST_TO_FPGA)
			? (src % CCIP_ALIGN_BYTES) : (dst % CCIP_ALIGN_BYTES);
		/* performing a short transfer to get aligned */
		if (alignment_offset != 0) {
			desc->rd_address = src & DMA_MASK_32_BIT;
			desc->wr_address = dst & DMA_MASK_32_BIT;
			desc->wr_burst_count = 1;
			desc->rd_burst_count = 1;
			desc->rd_address_ext = (src >> 32) & DMA_MASK_32_BIT;
			desc->wr_address_ext = (dst >> 32) & DMA_MASK_32_BIT;
			/* count isn't large enough to hit next 4CL boundary */
			if ((CCIP_ALIGN_BYTES - alignment_offset) >= count) {
				segment_size = count;
				count = 0;
			} else {
				segment_size = CCIP_ALIGN_BYTES
					- alignment_offset;
				src += segment_size;
				dst += segment_size;
				count -= segment_size;
				desc->control.transfer_irq_en = 0;
			}
			/* post short transfer to align to a 4CL (256 byte) */
			desc->len = segment_size;
			send_descriptor(ctx, desc);
		}
		/* at this point we are 4CL (256 byte) aligned */
		if (count >= CCIP_ALIGN_BYTES) {
			desc->rd_address = src & DMA_MASK_32_BIT;
			desc->wr_address = dst & DMA_MASK_32_BIT;
			desc->wr_burst_count = 4;
			desc->rd_burst_count = 4;
			desc->rd_address_ext = (src >> 32) & DMA_MASK_32_BIT;
			desc->wr_address_ext = (dst >> 32) & DMA_MASK_32_BIT;
			/* buffer ends on 4CL boundary */
			if ((count % CCIP_ALIGN_BYTES) == 0) {
				segment_size = count;
				count = 0;
			} else {
				segment_size = count
					- (count % CCIP_ALIGN_BYTES);
				src += segment_size;
				dst += segment_size;
				count -= segment_size;
				desc->control.transfer_irq_en = 0;
			}
			desc->len = segment_size;
			send_descriptor(ctx, desc);
		}
		/* post short transfer to handle the remainder */
		if (count > 0) {
			desc->rd_address = src & DMA_MASK_32_BIT;
			desc->wr_address = dst & DMA_MASK_32_BIT;
			desc->len = count;
			desc->wr_burst_count = 1;
			desc->rd_burst_count = 1;
			desc->rd_address_ext = (src >> 32) & DMA_MASK_32_BIT;
			desc->wr_address_ext = (dst >> 32) & DMA_MASK_32_BIT;
			if (intr_en)
				desc->control.transfer_irq_en = 1;
			send_descriptor(ctx, desc);
		}
	}

	return 0;
}

static int issue_magic(struct dma_afu_ctx *ctx)
{
	*(ctx->magic_buf) = 0ULL;
	return do_dma(ctx, DMA_WF_HOST_ADDR(ctx->magic_iova),
		DMA_WF_MAGIC_ROM, 64, 1, FPGA_TO_HOST, 1);
}

static void wait_magic(struct dma_afu_ctx *ctx)
{
	int magic_timeout = 0;

	if (!ctx)
		return;

	poll_interrupt(ctx);
	while (*(ctx->magic_buf) != DMA_WF_MAGIC) {
		if (magic_timeout++ > 1000) {
			IFPGA_RAWDEV_PMD_ERR("DMA magic operation timeout");
			magic_timeout = 0;
			break;
		}
	}
	*(ctx->magic_buf) = 0ULL;
}

static int dma_tx_buf(struct dma_afu_ctx *ctx, uint64_t dst, uint64_t src,
	uint64_t chunk, int is_last_chunk, int *intr_issued)
{
	int intr_en = 0;
	int ret = 0;

	if (!ctx || !intr_issued)
		return -EINVAL;

	src += chunk * ctx->dma_buf_size;
	dst += chunk * ctx->dma_buf_size;

	if (((chunk % HALF_DMA_BUF) == (HALF_DMA_BUF - 1)) || is_last_chunk) {
		if (*intr_issued) {
			ret = poll_interrupt(ctx);
			if (ret)
				return ret;
		}
		intr_en = 1;
	}

	chunk %= NUM_DMA_BUF;
	rte_memcpy(ctx->dma_buf[chunk], (void *)(uintptr_t)src,
		ctx->dma_buf_size);
	ret = do_dma(ctx, dst, DMA_HOST_ADDR(ctx->dma_iova[chunk]),
			ctx->dma_buf_size, 0, HOST_TO_FPGA, intr_en);
	if (intr_en)
		*intr_issued = 1;

	return ret;
}

static int dma_host_to_fpga(struct dma_afu_ctx *ctx, uint64_t dst, uint64_t src,
	size_t count)
{
	uint64_t i = 0;
	uint64_t count_left = count;
	uint64_t aligned_addr = 0;
	uint64_t align_bytes = 0;
	uint64_t dma_chunks = 0;
	uint64_t dma_tx_bytes = 0;
	uint64_t offset = 0;
	int issued_intr = 0;
	int ret = 0;

	IFPGA_RAWDEV_PMD_DEBUG("0x%"PRIx64" ---> 0x%"PRIx64" (%zu)", src, dst,
		count);

	if (!ctx)
		return -EINVAL;

	if (!IS_DMA_ALIGNED(dst)) {
		if (count_left < DMA_ALIGN_BYTES)
			return ase_host_to_fpga(ctx, &dst, &src, count_left);

		aligned_addr = ((dst / DMA_ALIGN_BYTES) + 1)
			* DMA_ALIGN_BYTES;
		align_bytes = aligned_addr - dst;
		ret = ase_host_to_fpga(ctx, &dst, &src, align_bytes);
		if (ret)
			return ret;
		count_left = count_left - align_bytes;
	}

	if (count_left) {
		dma_chunks = count_left / ctx->dma_buf_size;
		offset = dma_chunks * ctx->dma_buf_size;
		count_left -= offset;
		IFPGA_RAWDEV_PMD_DEBUG("0x%"PRIx64" ---> 0x%"PRIx64
			" (%"PRIu64"...0x%"PRIx64")",
			src, dst, dma_chunks, count_left);
		for (i = 0; i < dma_chunks; i++) {
			ret = dma_tx_buf(ctx, dst, src, i,
				i == (dma_chunks - 1), &issued_intr);
			if (ret)
				return ret;
		}

		if (issued_intr) {
			ret = poll_interrupt(ctx);
			if (ret)
				return ret;
		}

		if (count_left) {
			i = count_left / DMA_ALIGN_BYTES;
			if (i > 0) {
				dma_tx_bytes = i * DMA_ALIGN_BYTES;
				IFPGA_RAWDEV_PMD_DEBUG("left over 0x%"PRIx64" to DMA",
					dma_tx_bytes);
				rte_memcpy(ctx->dma_buf[0],
					(void *)(uintptr_t)(src + offset),
					dma_tx_bytes);
				ret = do_dma(ctx, dst + offset,
					DMA_HOST_ADDR(ctx->dma_iova[0]),
					dma_tx_bytes, 1, HOST_TO_FPGA, 1);
				if (ret)
					return ret;
				ret = poll_interrupt(ctx);
				if (ret)
					return ret;
			}

			count_left -= dma_tx_bytes;
			if (count_left) {
				IFPGA_RAWDEV_PMD_DEBUG("left over 0x%"PRIx64" to ASE",
					count_left);
				dst += offset + dma_tx_bytes;
				src += offset + dma_tx_bytes;
				ret = ase_host_to_fpga(ctx, &dst, &src,
					count_left);
			}
		}
	}

	return ret;
}

static int dma_rx_buf(struct dma_afu_ctx *ctx, uint64_t dst, uint64_t src,
	uint64_t chunk, int is_last_chunk, uint64_t *rx_count, int *wf_issued)
{
	uint64_t i = chunk % NUM_DMA_BUF;
	uint64_t n = *rx_count;
	uint64_t num_pending = 0;
	int ret = 0;

	if (!ctx || !wf_issued)
		return -EINVAL;

	ret = do_dma(ctx, DMA_HOST_ADDR(ctx->dma_iova[i]),
		src + chunk * ctx->dma_buf_size,
		ctx->dma_buf_size, 1, FPGA_TO_HOST, 0);
	if (ret)
		return ret;

	num_pending = chunk - n + 1;
	if (num_pending == HALF_DMA_BUF) {
		ret = issue_magic(ctx);
		if (ret) {
			IFPGA_RAWDEV_PMD_DEBUG("Magic issue failed");
			return ret;
		}
		*wf_issued = 1;
	}

	if ((num_pending > (NUM_DMA_BUF - 1)) || is_last_chunk) {
		if (*wf_issued) {
			wait_magic(ctx);
			for (i = 0; i < HALF_DMA_BUF; i++) {
				rte_memcpy((void *)(uintptr_t)(dst +
						n * ctx->dma_buf_size),
					ctx->dma_buf[n % NUM_DMA_BUF],
					ctx->dma_buf_size);
				n++;
			}
			*wf_issued = 0;
			*rx_count = n;
		}
		ret = issue_magic(ctx);
		if (ret) {
			IFPGA_RAWDEV_PMD_DEBUG("Magic issue failed");
			return ret;
		}
		*wf_issued = 1;
	}

	return ret;
}

static int dma_fpga_to_host(struct dma_afu_ctx *ctx, uint64_t dst, uint64_t src,
	size_t count)
{
	uint64_t i = 0;
	uint64_t count_left = count;
	uint64_t aligned_addr = 0;
	uint64_t align_bytes = 0;
	uint64_t dma_chunks = 0;
	uint64_t pending_buf = 0;
	uint64_t dma_rx_bytes = 0;
	uint64_t offset = 0;
	int wf_issued = 0;
	int ret = 0;

	IFPGA_RAWDEV_PMD_DEBUG("0x%"PRIx64" ---> 0x%"PRIx64" (%zu)", src, dst,
		count);

	if (!ctx)
		return -EINVAL;

	if (!IS_DMA_ALIGNED(src)) {
		if (count_left < DMA_ALIGN_BYTES)
			return ase_fpga_to_host(ctx, &src, &dst, count_left);

		aligned_addr = ((src / DMA_ALIGN_BYTES) + 1)
			 * DMA_ALIGN_BYTES;
		align_bytes = aligned_addr - src;
		ret = ase_fpga_to_host(ctx, &src, &dst, align_bytes);
		if (ret)
			return ret;
		count_left = count_left - align_bytes;
	}

	if (count_left) {
		dma_chunks = count_left / ctx->dma_buf_size;
		offset = dma_chunks * ctx->dma_buf_size;
		count_left -= offset;
		IFPGA_RAWDEV_PMD_DEBUG("0x%"PRIx64" ---> 0x%"PRIx64
			" (%"PRIu64"...0x%"PRIx64")",
			src, dst, dma_chunks, count_left);
		for (i = 0; i < dma_chunks; i++) {
			ret = dma_rx_buf(ctx, dst, src, i,
				i == (dma_chunks - 1),
				&pending_buf, &wf_issued);
			if (ret)
				return ret;
		}

		if (wf_issued)
			wait_magic(ctx);

		/* clear out final dma memcpy operations */
		while (pending_buf < dma_chunks) {
			/* constant size transfer; no length check required */
			rte_memcpy((void *)(uintptr_t)(dst +
					pending_buf * ctx->dma_buf_size),
				ctx->dma_buf[pending_buf % NUM_DMA_BUF],
				ctx->dma_buf_size);
			pending_buf++;
		}

		if (count_left > 0) {
			i = count_left / DMA_ALIGN_BYTES;
			if (i > 0) {
				dma_rx_bytes = i * DMA_ALIGN_BYTES;
				IFPGA_RAWDEV_PMD_DEBUG("left over 0x%"PRIx64" to DMA",
					dma_rx_bytes);
				ret = do_dma(ctx,
					DMA_HOST_ADDR(ctx->dma_iova[0]),
					src + offset,
					dma_rx_bytes, 1, FPGA_TO_HOST, 0);
				if (ret)
					return ret;
				ret = issue_magic(ctx);
				if (ret)
					return ret;
				wait_magic(ctx);
				rte_memcpy((void *)(uintptr_t)(dst + offset),
					ctx->dma_buf[0], dma_rx_bytes);
			}

			count_left -= dma_rx_bytes;
			if (count_left) {
				IFPGA_RAWDEV_PMD_DEBUG("left over 0x%"PRIx64" to ASE",
					count_left);
				dst += offset + dma_rx_bytes;
				src += offset + dma_rx_bytes;
				ret = ase_fpga_to_host(ctx, &src, &dst,
							count_left);
			}
		}
	}

	return ret;
}

static int dma_fpga_to_fpga(struct dma_afu_ctx *ctx, uint64_t dst, uint64_t src,
	size_t count)
{
	uint64_t i = 0;
	uint64_t count_left = count;
	uint64_t dma_chunks = 0;
	uint64_t offset = 0;
	uint32_t tx_chunks = 0;
	uint64_t *tmp_buf = NULL;
	int ret = 0;

	IFPGA_RAWDEV_PMD_DEBUG("0x%"PRIx64" ---> 0x%"PRIx64" (%zu)", src, dst,
		count);

	if (!ctx)
		return -EINVAL;

	if (IS_DMA_ALIGNED(dst) && IS_DMA_ALIGNED(src)
	    && IS_DMA_ALIGNED(count_left)) {
		dma_chunks = count_left / ctx->dma_buf_size;
		offset = dma_chunks * ctx->dma_buf_size;
		count_left -= offset;
		IFPGA_RAWDEV_PMD_DEBUG("0x%"PRIx64" ---> 0x%"PRIx64
			" (%"PRIu64"...0x%"PRIx64")",
			src, dst, dma_chunks, count_left);
		for (i = 0; i < dma_chunks; i++) {
			ret = do_dma(ctx, dst + i * ctx->dma_buf_size,
				src + i * ctx->dma_buf_size,
				ctx->dma_buf_size, 0, FPGA_TO_FPGA, 0);
			if (ret)
				return ret;
			if ((((i + 1) % NUM_DMA_BUF) == 0) ||
				(i == (dma_chunks - 1))) {
				ret = issue_magic(ctx);
				if (ret)
					return ret;
				wait_magic(ctx);
			}
		}

		if (count_left > 0) {
			IFPGA_RAWDEV_PMD_DEBUG("left over 0x%"PRIx64" to DMA", count_left);
			ret = do_dma(ctx, dst + offset, src + offset,
				count_left, 1, FPGA_TO_FPGA, 0);
			if (ret)
				return ret;
			ret = issue_magic(ctx);
			if (ret)
				return ret;
			wait_magic(ctx);
		}
	} else {
		if ((src < dst) && (src + count_left > dst)) {
			IFPGA_RAWDEV_PMD_ERR("Overlapping: 0x%"PRIx64
				" -> 0x%"PRIx64" (0x%"PRIx64")",
				src, dst, count_left);
			return -EINVAL;
		}
		tx_chunks = count_left / ctx->dma_buf_size;
		offset = tx_chunks * ctx->dma_buf_size;
		count_left -= offset;
		IFPGA_RAWDEV_PMD_DEBUG("0x%"PRIx64" --> 0x%"PRIx64
			" (%u...0x%"PRIx64")",
			src, dst, tx_chunks, count_left);
		tmp_buf = (uint64_t *)rte_malloc(NULL, ctx->dma_buf_size,
			DMA_ALIGN_BYTES);
		for (i = 0; i < tx_chunks; i++) {
			ret = dma_fpga_to_host(ctx, (uint64_t)tmp_buf,
				src + i * ctx->dma_buf_size,
				ctx->dma_buf_size);
			if (ret)
				goto free_buf;
			ret = dma_host_to_fpga(ctx,
				dst + i * ctx->dma_buf_size,
				(uint64_t)tmp_buf, ctx->dma_buf_size);
			if (ret)
				goto free_buf;
		}

		if (count_left > 0) {
			ret = dma_fpga_to_host(ctx, (uint64_t)tmp_buf,
				src + offset, count_left);
			if (ret)
				goto free_buf;
			ret = dma_host_to_fpga(ctx, dst + offset,
				(uint64_t)tmp_buf, count_left);
			if (ret)
				goto free_buf;
		}
free_buf:
		rte_free(tmp_buf);
	}

	return ret;
}

static int dma_transfer_sync(struct dma_afu_ctx *ctx, uint64_t dst,
	uint64_t src, size_t count, fpga_dma_type type)
{
	int ret = 0;

	if (!ctx)
		return -EINVAL;

	if (type == HOST_TO_FPGA)
		ret = dma_host_to_fpga(ctx, dst, src, count);
	else if (type == FPGA_TO_HOST)
		ret = dma_fpga_to_host(ctx, dst, src, count);
	else if (type == FPGA_TO_FPGA)
		ret = dma_fpga_to_fpga(ctx, dst, src, count);
	else
		return -EINVAL;

	return ret;
}

static double get_duration(struct timespec start, struct timespec end)
{
	uint64_t diff = 1000000000L * (end.tv_sec - start.tv_sec)
		+ end.tv_nsec - start.tv_nsec;
	return (double)diff / (double)1000000000L;
}

#define SWEEP_ITERS 1
static int sweep_test(struct dma_afu_ctx *ctx, uint32_t length,
	uint64_t ddr_offset, uint64_t buf_offset, uint64_t size_decrement)
{
	struct timespec start, end;
	uint64_t test_size = 0;
	uint64_t *dma_buf_ptr = NULL;
	double throughput, total_time = 0.0;
	int i = 0;
	int ret = 0;

	if (!ctx || !ctx->data_buf || !ctx->ref_buf) {
		IFPGA_RAWDEV_PMD_ERR("Buffer for DMA test is not allocated");
		return -EINVAL;
	}

	if (length < (buf_offset + size_decrement)) {
		IFPGA_RAWDEV_PMD_ERR("Test length does not match unaligned parameter");
		return -EINVAL;
	}
	test_size = length - (buf_offset + size_decrement);
	if ((ddr_offset + test_size) > ctx->mem_size) {
		IFPGA_RAWDEV_PMD_ERR("Test is out of DDR memory space");
		return -EINVAL;
	}

	dma_buf_ptr = (uint64_t *)((uint8_t *)ctx->data_buf + buf_offset);
	printf("Sweep Host %p to FPGA 0x%"PRIx64
		" with 0x%"PRIx64" bytes ...\n",
		(void *)dma_buf_ptr, ddr_offset, test_size);

	for (i = 0; i < SWEEP_ITERS; i++) {
		clock_gettime(CLOCK_MONOTONIC, &start);
		ret = dma_transfer_sync(ctx, ddr_offset, (uint64_t)dma_buf_ptr,
			test_size, HOST_TO_FPGA);
		clock_gettime(CLOCK_MONOTONIC, &end);
		if (ret) {
			IFPGA_RAWDEV_PMD_ERR("Failed");
			return ret;
		}
		total_time += get_duration(start, end);
	}
	throughput = (test_size * SWEEP_ITERS) / (total_time * 1000000);
	printf("Measured bandwidth = %lf MB/s\n", throughput);

	printf("Sweep FPGA 0x%"PRIx64" to Host %p with 0x%"PRIx64" bytes ...\n",
		ddr_offset, (void *)dma_buf_ptr, test_size);

	total_time = 0.0;
	memset((char *)dma_buf_ptr, 0, test_size);
	for (i = 0; i < SWEEP_ITERS; i++) {
		clock_gettime(CLOCK_MONOTONIC, &start);
		ret = dma_transfer_sync(ctx, (uint64_t)dma_buf_ptr, ddr_offset,
			test_size, FPGA_TO_HOST);
		clock_gettime(CLOCK_MONOTONIC, &end);
		if (ret) {
			IFPGA_RAWDEV_PMD_ERR("Failed");
			return ret;
		}
		total_time += get_duration(start, end);
	}
	throughput = (test_size * SWEEP_ITERS) / (total_time * 1000000);
	printf("Measured bandwidth = %lf MB/s\n", throughput);

	printf("Verifying buffer ...\n");
	return dma_afu_buf_verify(ctx, test_size);
}

static int dma_afu_test(struct afu_rawdev *dev)
{
	struct n3000_afu_priv *priv = NULL;
	struct dma_afu_ctx *ctx = NULL;
	struct rte_pmd_afu_dma_cfg *cfg = NULL;
	msgdma_ctrl ctrl;
	uint64_t offset = 0;
	uint32_t i = 0;
	int ret = 0;

	if (!dev)
		return -EINVAL;

	if (!dev->priv)
		return -ENOENT;

	priv = (struct n3000_afu_priv *)dev->priv;
	cfg = &priv->dma_cfg;
	if (cfg->index >= NUM_N3000_DMA)
		return -EINVAL;
	ctx = &priv->dma_ctx[cfg->index];

	ctx->pattern = (int)cfg->pattern;
	ctx->verbose = (int)cfg->verbose;
	ctx->dma_buf_size = cfg->size;

	ret = dma_afu_buf_alloc(ctx, cfg);
	if (ret)
		goto free;

	printf("Initialize test buffer\n");
	dma_afu_buf_init(ctx, cfg->length);

	/* enable interrupt */
	ctrl.csr = 0;
	ctrl.global_intr_en_mask = 1;
	rte_write32(ctrl.csr, CSR_CONTROL(ctx->csr_addr));

	printf("Host %p to FPGA 0x%x with 0x%x bytes\n", ctx->data_buf,
		cfg->offset, cfg->length);
	ret = dma_transfer_sync(ctx, cfg->offset, (uint64_t)ctx->data_buf,
		cfg->length, HOST_TO_FPGA);
	if (ret) {
		IFPGA_RAWDEV_PMD_ERR("Failed to transfer data from host to FPGA");
		goto end;
	}
	memset(ctx->data_buf, 0, cfg->length);

	printf("FPGA 0x%x to Host %p with 0x%x bytes\n", cfg->offset,
		ctx->data_buf, cfg->length);
	ret = dma_transfer_sync(ctx, (uint64_t)ctx->data_buf, cfg->offset,
		cfg->length, FPGA_TO_HOST);
	if (ret) {
		IFPGA_RAWDEV_PMD_ERR("Failed to transfer data from FPGA to host");
		goto end;
	}
	ret = dma_afu_buf_verify(ctx, cfg->length);
	if (ret)
		goto end;

	if ((cfg->offset + cfg->length * 2) <= ctx->mem_size)
		offset = cfg->offset + cfg->length;
	else if (cfg->offset > cfg->length)
		offset = 0;
	else
		goto end;

	printf("FPGA 0x%x to FPGA 0x%"PRIx64" with 0x%x bytes\n",
		cfg->offset, offset, cfg->length);
	ret = dma_transfer_sync(ctx, offset, cfg->offset, cfg->length,
		FPGA_TO_FPGA);
	if (ret) {
		IFPGA_RAWDEV_PMD_ERR("Failed to transfer data from FPGA to FPGA");
		goto end;
	}

	printf("FPGA 0x%"PRIx64" to Host %p with 0x%x bytes\n", offset,
		ctx->data_buf, cfg->length);
	ret = dma_transfer_sync(ctx, (uint64_t)ctx->data_buf, offset,
		cfg->length, FPGA_TO_HOST);
	if (ret) {
		IFPGA_RAWDEV_PMD_ERR("Failed to transfer data from FPGA to host");
		goto end;
	}
	ret = dma_afu_buf_verify(ctx, cfg->length);
	if (ret)
		goto end;

	printf("Sweep with aligned address and size\n");
	ret = sweep_test(ctx, cfg->length, cfg->offset, 0, 0);
	if (ret)
		goto end;

	if (cfg->unaligned) {
		printf("Sweep with unaligned address and size\n");
		struct unaligned_set {
			uint64_t addr_offset;
			uint64_t size_dec;
		} param[] = {{61, 5}, {3, 0}, {7, 3}, {0, 3}, {0, 61}, {0, 7}};
		for (i = 0; i < ARRAY_SIZE(param); i++) {
			ret = sweep_test(ctx, cfg->length, cfg->offset,
				param[i].addr_offset, param[i].size_dec);
			if (ret)
				break;
		}
	}

end:
	/* disable interrupt */
	ctrl.global_intr_en_mask = 0;
	rte_write32(ctrl.csr, CSR_CONTROL(ctx->csr_addr));

free:
	dma_afu_buf_free(ctx);
	return ret;
}

static struct rte_pci_device *n3000_afu_get_pci_dev(struct afu_rawdev *dev)
{
	struct rte_afu_device *afudev = NULL;

	if (!dev || !dev->rawdev || !dev->rawdev->device)
		return NULL;

	afudev = RTE_DEV_TO_AFU(dev->rawdev->device);
	if (!afudev->rawdev || !afudev->rawdev->device)
		return NULL;

	return RTE_DEV_TO_PCI(afudev->rawdev->device);
}

#ifdef VFIO_PRESENT
static int dma_afu_set_irqs(struct afu_rawdev *dev, uint32_t vec_start,
	uint32_t count, int *efds)
{
	struct rte_pci_device *pci_dev = NULL;
	struct vfio_irq_set *irq_set = NULL;
	int vfio_dev_fd = 0;
	size_t sz = 0;
	int ret = 0;

	if (!dev || !efds || (count == 0) || (count > MAX_MSIX_VEC))
		return -EINVAL;

	pci_dev = n3000_afu_get_pci_dev(dev);
	if (!pci_dev)
		return -ENODEV;
	vfio_dev_fd = rte_intr_dev_fd_get(pci_dev->intr_handle);

	sz = sizeof(*irq_set) + sizeof(*efds) * count;
	irq_set = rte_zmalloc(NULL, sz, 0);
	if (!irq_set)
		return -ENOMEM;

	irq_set->argsz = (uint32_t)sz;
	irq_set->count = count;
	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
		VFIO_IRQ_SET_ACTION_TRIGGER;
	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
	irq_set->start = vec_start;

	rte_memcpy(&irq_set->data, efds, sizeof(*efds) * count);
	ret = ioctl(vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
	if (ret)
		IFPGA_RAWDEV_PMD_ERR("Error enabling MSI-X interrupts\n");

	rte_free(irq_set);
	return ret;
}
#endif

static void *n3000_afu_get_port_addr(struct afu_rawdev *dev)
{
	struct rte_pci_device *pci_dev = NULL;
	uint8_t *addr = NULL;
	uint64_t val = 0;
	uint32_t bar = 0;

	pci_dev = n3000_afu_get_pci_dev(dev);
	if (!pci_dev)
		return NULL;

	addr = (uint8_t *)pci_dev->mem_resource[0].addr;
	val = rte_read64(addr + PORT_ATTR_REG(dev->port));
	if (!PORT_IMPLEMENTED(val)) {
		IFPGA_RAWDEV_PMD_INFO("FIU port %d is not implemented", dev->port);
		return NULL;
	}

	bar = PORT_BAR(val);
	if (bar >= PCI_MAX_RESOURCE) {
		IFPGA_RAWDEV_PMD_ERR("BAR index %u is out of limit", bar);
		return NULL;
	}

	addr = (uint8_t *)pci_dev->mem_resource[bar].addr + PORT_OFFSET(val);
	return addr;
}

static int n3000_afu_get_irq_capability(struct afu_rawdev *dev,
	uint32_t *vec_start, uint32_t *vec_count)
{
	uint8_t *addr = NULL;
	uint64_t val = 0;
	uint64_t header = 0;
	uint64_t next_offset = 0;

	addr = (uint8_t *)n3000_afu_get_port_addr(dev);
	if (!addr)
		return -ENOENT;

	do {
		addr += next_offset;
		header = rte_read64(addr);
		if ((DFH_TYPE(header) == DFH_TYPE_PRIVATE) &&
			(DFH_FEATURE_ID(header) == PORT_FEATURE_UINT_ID)) {
			val = rte_read64(addr + PORT_UINT_CAP_REG);
			if (vec_start)
				*vec_start = PORT_VEC_START(val);
			if (vec_count)
				*vec_count = PORT_VEC_COUNT(val);
			return 0;
		}
		next_offset = DFH_NEXT_OFFSET(header);
		if (((next_offset & 0xffff) == 0xffff) || (next_offset == 0))
			break;
	} while (!DFH_EOL(header));

	return -ENOENT;
}

static int nlb_afu_ctx_release(struct afu_rawdev *dev)
{
	struct n3000_afu_priv *priv = NULL;
	struct nlb_afu_ctx *ctx = NULL;

	if (!dev)
		return -EINVAL;

	priv = (struct n3000_afu_priv *)dev->priv;
	if (!priv)
		return -ENOENT;

	ctx = &priv->nlb_ctx;

	rte_free(ctx->dsm_ptr);
	ctx->dsm_ptr = NULL;
	ctx->status_ptr = NULL;

	rte_free(ctx->src_ptr);
	ctx->src_ptr = NULL;

	rte_free(ctx->dest_ptr);
	ctx->dest_ptr = NULL;

	return 0;
}

static int nlb_afu_ctx_init(struct afu_rawdev *dev, uint8_t *addr)
{
	struct n3000_afu_priv *priv = NULL;
	struct nlb_afu_ctx *ctx = NULL;
	int ret = 0;

	if (!dev || !addr)
		return -EINVAL;

	priv = (struct n3000_afu_priv *)dev->priv;
	if (!priv)
		return -ENOENT;

	ctx = &priv->nlb_ctx;
	ctx->addr = addr;

	ctx->dsm_ptr = (uint8_t *)rte_zmalloc(NULL, DSM_SIZE, TEST_MEM_ALIGN);
	if (!ctx->dsm_ptr)
		return -ENOMEM;

	ctx->dsm_iova = rte_malloc_virt2iova(ctx->dsm_ptr);
	if (ctx->dsm_iova == RTE_BAD_IOVA) {
		ret = -ENOMEM;
		goto release_dsm;
	}

	ctx->src_ptr = (uint8_t *)rte_zmalloc(NULL, NLB_BUF_SIZE,
		TEST_MEM_ALIGN);
	if (!ctx->src_ptr) {
		ret = -ENOMEM;
		goto release_dsm;
	}
	ctx->src_iova = rte_malloc_virt2iova(ctx->src_ptr);
	if (ctx->src_iova == RTE_BAD_IOVA) {
		ret = -ENOMEM;
		goto release_src;
	}

	ctx->dest_ptr = (uint8_t *)rte_zmalloc(NULL, NLB_BUF_SIZE,
		TEST_MEM_ALIGN);
	if (!ctx->dest_ptr) {
		ret = -ENOMEM;
		goto release_src;
	}
	ctx->dest_iova = rte_malloc_virt2iova(ctx->dest_ptr);
	if (ctx->dest_iova == RTE_BAD_IOVA) {
		ret = -ENOMEM;
		goto release_dest;
	}

	ctx->status_ptr = (struct nlb_dsm_status *)(ctx->dsm_ptr + DSM_STATUS);
	return 0;

release_dest:
	rte_free(ctx->dest_ptr);
	ctx->dest_ptr = NULL;
release_src:
	rte_free(ctx->src_ptr);
	ctx->src_ptr = NULL;
release_dsm:
	rte_free(ctx->dsm_ptr);
	ctx->dsm_ptr = NULL;
	return ret;
}

static int dma_afu_ctx_release(struct afu_rawdev *dev)
{
	struct n3000_afu_priv *priv = NULL;
	struct dma_afu_ctx *ctx = NULL;

	if (!dev)
		return -EINVAL;

	priv = (struct n3000_afu_priv *)dev->priv;
	if (!priv)
		return -ENOENT;

	ctx = &priv->dma_ctx[0];

	rte_free(ctx->desc_buf);
	ctx->desc_buf = NULL;

	rte_free(ctx->magic_buf);
	ctx->magic_buf = NULL;

	close(ctx->event_fd);
	return 0;
}

static int dma_afu_ctx_init(struct afu_rawdev *dev, int index, uint8_t *addr)
{
	struct n3000_afu_priv *priv = NULL;
	struct dma_afu_ctx *ctx = NULL;
	uint64_t mem_sz[] = {0x100000000, 0x100000000, 0x40000000, 0x1000000};
	static int efds[1] = {0};
	uint32_t vec_start = 0;
	int ret = 0;

	if (!dev || (index < 0) || (index >= NUM_N3000_DMA) || !addr)
		return -EINVAL;

	priv = (struct n3000_afu_priv *)dev->priv;
	if (!priv)
		return -ENOENT;

	ctx = &priv->dma_ctx[index];
	ctx->index = index;
	ctx->addr = addr;
	ctx->csr_addr = addr + DMA_CSR;
	ctx->desc_addr = addr + DMA_DESC;
	ctx->ase_ctrl_addr = addr + DMA_ASE_CTRL;
	ctx->ase_data_addr = addr + DMA_ASE_DATA;
	ctx->mem_size = mem_sz[ctx->index];
	ctx->cur_ase_page = INVALID_ASE_PAGE;
	if (ctx->index == 0) {
		ret = n3000_afu_get_irq_capability(dev, &vec_start, NULL);
		if (ret)
			return ret;

		efds[0] = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
		if (efds[0] < 0) {
			IFPGA_RAWDEV_PMD_ERR("eventfd create failed");
			return -EBADF;
		}
#ifdef VFIO_PRESENT
		if (dma_afu_set_irqs(dev, vec_start, 1, efds))
			IFPGA_RAWDEV_PMD_ERR("DMA interrupt setup failed");
#endif
	}
	ctx->event_fd = efds[0];

	ctx->desc_buf = (msgdma_ext_desc *)rte_zmalloc(NULL,
		sizeof(msgdma_ext_desc), DMA_ALIGN_BYTES);
	if (!ctx->desc_buf) {
		ret = -ENOMEM;
		goto release;
	}

	ctx->magic_buf = (uint64_t *)rte_zmalloc(NULL, MAGIC_BUF_SIZE,
		TEST_MEM_ALIGN);
	if (!ctx->magic_buf) {
		ret = -ENOMEM;
		goto release;
	}
	ctx->magic_iova = rte_malloc_virt2iova(ctx->magic_buf);
	if (ctx->magic_iova == RTE_BAD_IOVA) {
		ret = -ENOMEM;
		goto release;
	}

	return 0;

release:
	dma_afu_ctx_release(dev);
	return ret;
}

static int n3000_afu_ctx_init(struct afu_rawdev *dev)
{
	struct n3000_afu_priv *priv = NULL;
	uint8_t *addr = NULL;
	uint64_t header = 0;
	uint64_t uuid_hi = 0;
	uint64_t uuid_lo = 0;
	uint64_t next_offset = 0;
	int ret = 0;

	if (!dev)
		return -EINVAL;

	priv = (struct n3000_afu_priv *)dev->priv;
	if (!priv)
		return -ENOENT;

	addr = (uint8_t *)dev->addr;
	do {
		addr += next_offset;
		header = rte_read64(addr);
		uuid_lo = rte_read64(addr + DFH_UUID_L_OFFSET);
		uuid_hi = rte_read64(addr + DFH_UUID_H_OFFSET);

		if ((DFH_TYPE(header) == DFH_TYPE_AFU) &&
			(uuid_lo == N3000_NLB0_UUID_L) &&
			(uuid_hi == N3000_NLB0_UUID_H)) {
			IFPGA_RAWDEV_PMD_INFO("AFU NLB0 found @ %p", (void *)addr);
			ret = nlb_afu_ctx_init(dev, addr);
			if (ret)
				return ret;
		} else if ((DFH_TYPE(header) == DFH_TYPE_BBB) &&
			(uuid_lo == N3000_DMA_UUID_L) &&
			(uuid_hi == N3000_DMA_UUID_H) &&
			(priv->num_dma < NUM_N3000_DMA)) {
			IFPGA_RAWDEV_PMD_INFO("AFU DMA%d found @ %p",
				priv->num_dma, (void *)addr);
			ret = dma_afu_ctx_init(dev, priv->num_dma, addr);
			if (ret)
				return ret;
			priv->num_dma++;
		} else {
			IFPGA_RAWDEV_PMD_DEBUG("DFH: type %"PRIu64
				", uuid %016"PRIx64"%016"PRIx64,
				DFH_TYPE(header), uuid_hi, uuid_lo);
		}

		next_offset = DFH_NEXT_OFFSET(header);
		if (((next_offset & 0xffff) == 0xffff) || (next_offset == 0))
			break;
	} while (!DFH_EOL(header));

	return 0;
}

static int n3000_afu_init(struct afu_rawdev *dev)
{
	if (!dev)
		return -EINVAL;

	if (!dev->priv) {
		dev->priv = rte_zmalloc(NULL, sizeof(struct n3000_afu_priv), 0);
		if (!dev->priv)
			return -ENOMEM;
	}

	return n3000_afu_ctx_init(dev);
}

static int n3000_afu_config(struct afu_rawdev *dev, void *config,
	size_t config_size)
{
	struct n3000_afu_priv *priv = NULL;
	struct rte_pmd_afu_n3000_cfg *cfg = NULL;
	int i = 0;
	uint64_t top = 0;

	if (!dev || !config || !config_size)
		return -EINVAL;

	priv = (struct n3000_afu_priv *)dev->priv;
	if (!priv)
		return -ENOENT;

	if (config_size != sizeof(struct rte_pmd_afu_n3000_cfg))
		return -EINVAL;

	cfg = (struct rte_pmd_afu_n3000_cfg *)config;
	if (cfg->type == RTE_PMD_AFU_N3000_NLB) {
		if (cfg->nlb_cfg.mode != NLB_MODE_LPBK)
			return -EINVAL;
		if ((cfg->nlb_cfg.read_vc > NLB_VC_RANDOM) ||
			(cfg->nlb_cfg.write_vc > NLB_VC_RANDOM))
			return -EINVAL;
		if (cfg->nlb_cfg.wrfence_vc > NLB_VC_VH1)
			return -EINVAL;
		if (cfg->nlb_cfg.cache_hint > NLB_RDLINE_MIXED)
			return -EINVAL;
		if (cfg->nlb_cfg.cache_policy > NLB_WRPUSH_I)
			return -EINVAL;
		if ((cfg->nlb_cfg.multi_cl != 1) &&
			(cfg->nlb_cfg.multi_cl != 2) &&
			(cfg->nlb_cfg.multi_cl != 4))
			return -EINVAL;
		if ((cfg->nlb_cfg.begin < MIN_CACHE_LINES) ||
			(cfg->nlb_cfg.begin > MAX_CACHE_LINES))
			return -EINVAL;
		if ((cfg->nlb_cfg.end < cfg->nlb_cfg.begin) ||
			(cfg->nlb_cfg.end > MAX_CACHE_LINES))
			return -EINVAL;
		rte_memcpy(&priv->nlb_cfg, &cfg->nlb_cfg,
			sizeof(struct rte_pmd_afu_nlb_cfg));
	} else if (cfg->type == RTE_PMD_AFU_N3000_DMA) {
		if (cfg->dma_cfg.index >= NUM_N3000_DMA)
			return -EINVAL;
		i = cfg->dma_cfg.index;
		if (cfg->dma_cfg.length > priv->dma_ctx[i].mem_size)
			return -EINVAL;
		if (cfg->dma_cfg.offset >= priv->dma_ctx[i].mem_size)
			return -EINVAL;
		top = cfg->dma_cfg.length + cfg->dma_cfg.offset;
		if ((top == 0) || (top > priv->dma_ctx[i].mem_size))
			return -EINVAL;
		if (i == 3) {  /* QDR connected to DMA3 */
			if (cfg->dma_cfg.length & 0x3f) {
				cfg->dma_cfg.length &= ~0x3f;
				IFPGA_RAWDEV_PMD_INFO("Round size to %x for QDR",
					cfg->dma_cfg.length);
			}
		}
		rte_memcpy(&priv->dma_cfg, &cfg->dma_cfg,
			sizeof(struct rte_pmd_afu_dma_cfg));
	} else {
		IFPGA_RAWDEV_PMD_ERR("Invalid type of N3000 AFU");
		return -EINVAL;
	}

	priv->cfg_type = cfg->type;
	return 0;
}

static int n3000_afu_test(struct afu_rawdev *dev)
{
	struct n3000_afu_priv *priv = NULL;
	int ret = 0;

	if (!dev)
		return -EINVAL;

	if (!dev->priv)
		return -ENOENT;

	priv = (struct n3000_afu_priv *)dev->priv;

	if (priv->cfg_type == RTE_PMD_AFU_N3000_NLB) {
		IFPGA_RAWDEV_PMD_INFO("Test NLB");
		ret = nlb_afu_test(dev);
	} else if (priv->cfg_type == RTE_PMD_AFU_N3000_DMA) {
		IFPGA_RAWDEV_PMD_INFO("Test DMA%u", priv->dma_cfg.index);
		ret = dma_afu_test(dev);
	} else {
		IFPGA_RAWDEV_PMD_ERR("Please configure AFU before test");
		ret = -EINVAL;
	}

	return ret;
}

static int n3000_afu_close(struct afu_rawdev *dev)
{
	if (!dev)
		return -EINVAL;

	nlb_afu_ctx_release(dev);
	dma_afu_ctx_release(dev);

	rte_free(dev->priv);
	dev->priv = NULL;

	return 0;
}

static int n3000_afu_dump(struct afu_rawdev *dev, FILE *f)
{
	struct n3000_afu_priv *priv = NULL;

	if (!dev)
		return -EINVAL;

	priv = (struct n3000_afu_priv *)dev->priv;
	if (!priv)
		return -ENOENT;

	if (!f)
		f = stdout;

	if (priv->cfg_type == RTE_PMD_AFU_N3000_NLB) {
		struct nlb_afu_ctx *ctx = &priv->nlb_ctx;
		fprintf(f, "addr:\t\t%p\n", (void *)ctx->addr);
		fprintf(f, "dsm_ptr:\t%p\n", (void *)ctx->dsm_ptr);
		fprintf(f, "dsm_iova:\t0x%"PRIx64"\n", ctx->dsm_iova);
		fprintf(f, "src_ptr:\t%p\n", (void *)ctx->src_ptr);
		fprintf(f, "src_iova:\t0x%"PRIx64"\n", ctx->src_iova);
		fprintf(f, "dest_ptr:\t%p\n", (void *)ctx->dest_ptr);
		fprintf(f, "dest_iova:\t0x%"PRIx64"\n", ctx->dest_iova);
		fprintf(f, "status_ptr:\t%p\n", (void *)ctx->status_ptr);
	} else if (priv->cfg_type == RTE_PMD_AFU_N3000_DMA) {
		struct dma_afu_ctx *ctx = &priv->dma_ctx[priv->dma_cfg.index];
		fprintf(f, "index:\t\t%d\n", ctx->index);
		fprintf(f, "addr:\t\t%p\n", (void *)ctx->addr);
		fprintf(f, "csr_addr:\t%p\n", (void *)ctx->csr_addr);
		fprintf(f, "desc_addr:\t%p\n", (void *)ctx->desc_addr);
		fprintf(f, "ase_ctrl_addr:\t%p\n", (void *)ctx->ase_ctrl_addr);
		fprintf(f, "ase_data_addr:\t%p\n", (void *)ctx->ase_data_addr);
		fprintf(f, "desc_buf:\t%p\n", (void *)ctx->desc_buf);
		fprintf(f, "magic_buf:\t%p\n", (void *)ctx->magic_buf);
		fprintf(f, "magic_iova:\t0x%"PRIx64"\n", ctx->magic_iova);
	} else {
		return -EINVAL;
	}

	return 0;
}

static int n3000_afu_reset(struct afu_rawdev *dev)
{
	uint8_t *addr = NULL;
	uint64_t val = 0;

	addr = (uint8_t *)n3000_afu_get_port_addr(dev);
	if (!addr)
		return -ENOENT;

	val = rte_read64(addr + PORT_CTRL_REG);
	val |= PORT_SOFT_RESET;
	rte_write64(val, addr + PORT_CTRL_REG);
	rte_delay_us(100);
	val &= ~PORT_SOFT_RESET;
	rte_write64(val, addr + PORT_CTRL_REG);

	return 0;
}

static struct afu_ops n3000_afu_ops = {
	.init = n3000_afu_init,
	.config = n3000_afu_config,
	.start = NULL,
	.stop = NULL,
	.test = n3000_afu_test,
	.close = n3000_afu_close,
	.dump = n3000_afu_dump,
	.reset = n3000_afu_reset
};

static struct afu_rawdev_drv n3000_afu_drv = {
	.uuid = { N3000_AFU_UUID_L, N3000_AFU_UUID_H },
	.ops = &n3000_afu_ops
};

AFU_PMD_REGISTER(n3000_afu_drv);