numam-dpdk/examples/vhost/ioat.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(c) 2010-2020 Intel Corporation
 */

#include <sys/uio.h>
#ifdef RTE_RAW_IOAT
#include <rte_rawdev.h>
#include <rte_ioat_rawdev.h>

#include "ioat.h"
#include "main.h"

struct dma_for_vhost dma_bind[MAX_VHOST_DEVICE];

struct packet_tracker {
	unsigned short size_track[MAX_ENQUEUED_SIZE];
	unsigned short next_read;
	unsigned short next_write;
	unsigned short last_remain;
	unsigned short ioat_space;
};

struct packet_tracker cb_tracker[MAX_VHOST_DEVICE];

int
open_ioat(const char *value)
{
	struct dma_for_vhost *dma_info = dma_bind;
	char *input = strndup(value, strlen(value) + 1);
	char *addrs = input;
	char *ptrs[2];
	char *start, *end, *substr;
	int64_t vid, vring_id;
	struct rte_ioat_rawdev_config config;
	struct rte_rawdev_info info = { .dev_private = &config };
	char name[32];
	int dev_id;
	int ret = 0;
	uint16_t i = 0;
	char *dma_arg[MAX_VHOST_DEVICE];
	int args_nr;

	while (isblank(*addrs))
		addrs++;
	if (*addrs == '\0') {
		ret = -1;
		goto out;
	}

	/* process DMA devices within bracket. */
	addrs++;
	substr = strtok(addrs, ";]");
	if (!substr) {
		ret = -1;
		goto out;
	}
	args_nr = rte_strsplit(substr, strlen(substr),
			dma_arg, MAX_VHOST_DEVICE, ',');
	if (args_nr <= 0) {
		ret = -1;
		goto out;
	}
	while (i < args_nr) {
		char *arg_temp = dma_arg[i];
		uint8_t sub_nr;
		sub_nr = rte_strsplit(arg_temp, strlen(arg_temp), ptrs, 2, '@');
		if (sub_nr != 2) {
			ret = -1;
			goto out;
		}

		start = strstr(ptrs[0], "txd");
		if (start == NULL) {
			ret = -1;
			goto out;
		}

		start += 3;
		vid = strtol(start, &end, 0);
		if (end == start) {
			ret = -1;
			goto out;
		}

		vring_id = 0 + VIRTIO_RXQ;
		if (rte_pci_addr_parse(ptrs[1],
				&(dma_info + vid)->dmas[vring_id].addr) < 0) {
			ret = -1;
			goto out;
		}

		rte_pci_device_name(&(dma_info + vid)->dmas[vring_id].addr,
				name, sizeof(name));
		dev_id = rte_rawdev_get_dev_id(name);
		if (dev_id == (uint16_t)(-ENODEV) ||
		dev_id == (uint16_t)(-EINVAL)) {
			ret = -1;
			goto out;
		}

		if (rte_rawdev_info_get(dev_id, &info, sizeof(config)) < 0 ||
		strstr(info.driver_name, "ioat") == NULL) {
			ret = -1;
			goto out;
		}

		(dma_info + vid)->dmas[vring_id].dev_id = dev_id;
		(dma_info + vid)->dmas[vring_id].is_valid = true;
		config.ring_size = IOAT_RING_SIZE;
		config.hdls_disable = true;
		if (rte_rawdev_configure(dev_id, &info, sizeof(config)) < 0) {
			ret = -1;
			goto out;
		}
		rte_rawdev_start(dev_id);
		cb_tracker[dev_id].ioat_space = IOAT_RING_SIZE - 1;
		dma_info->nr++;
		i++;
	}
out:
	free(input);
	return ret;
}

uint32_t
ioat_transfer_data_cb(int vid, uint16_t queue_id,
		struct rte_vhost_async_desc *descs,
		struct rte_vhost_async_status *opaque_data, uint16_t count)
{
	uint32_t i_desc;
	uint16_t dev_id = dma_bind[vid].dmas[queue_id * 2 + VIRTIO_RXQ].dev_id;
	struct rte_vhost_iov_iter *src = NULL;
	struct rte_vhost_iov_iter *dst = NULL;
	unsigned long i_seg;
	unsigned short mask = MAX_ENQUEUED_SIZE - 1;
	unsigned short write = cb_tracker[dev_id].next_write;

	if (!opaque_data) {
		for (i_desc = 0; i_desc < count; i_desc++) {
			src = descs[i_desc].src;
			dst = descs[i_desc].dst;
			i_seg = 0;
			if (cb_tracker[dev_id].ioat_space < src->nr_segs)
				break;
			while (i_seg < src->nr_segs) {
				rte_ioat_enqueue_copy(dev_id,
					(uintptr_t)(src->iov[i_seg].iov_base)
						+ src->offset,
					(uintptr_t)(dst->iov[i_seg].iov_base)
						+ dst->offset,
					src->iov[i_seg].iov_len,
					0,
					0);
				i_seg++;
			}
			write &= mask;
			cb_tracker[dev_id].size_track[write] = src->nr_segs;
			cb_tracker[dev_id].ioat_space -= src->nr_segs;
			write++;
		}
	} else {
		/* Opaque data is not supported */
		return -1;
	}
	/* ring the doorbell */
	rte_ioat_perform_ops(dev_id);
	cb_tracker[dev_id].next_write = write;
	return i_desc;
}

uint32_t
ioat_check_completed_copies_cb(int vid, uint16_t queue_id,
		struct rte_vhost_async_status *opaque_data,
		uint16_t max_packets)
{
	if (!opaque_data) {
		uintptr_t dump[255];
		int n_seg;
		unsigned short read, write;
		unsigned short nb_packet = 0;
		unsigned short mask = MAX_ENQUEUED_SIZE - 1;
		unsigned short i;

		uint16_t dev_id = dma_bind[vid].dmas[queue_id * 2
				+ VIRTIO_RXQ].dev_id;
		n_seg = rte_ioat_completed_ops(dev_id, 255, NULL, NULL, dump, dump);
		if (n_seg < 0) {
			RTE_LOG(ERR,
				VHOST_DATA,
				"fail to poll completed buf on IOAT device %u",
				dev_id);
			return 0;
		}
		if (n_seg == 0)
			return 0;

		cb_tracker[dev_id].ioat_space += n_seg;
		n_seg += cb_tracker[dev_id].last_remain;

		read = cb_tracker[dev_id].next_read;
		write = cb_tracker[dev_id].next_write;
		for (i = 0; i < max_packets; i++) {
			read &= mask;
			if (read == write)
				break;
			if (n_seg >= cb_tracker[dev_id].size_track[read]) {
				n_seg -= cb_tracker[dev_id].size_track[read];
				read++;
				nb_packet++;
			} else {
				break;
			}
		}
		cb_tracker[dev_id].next_read = read;
		cb_tracker[dev_id].last_remain = n_seg;
		return nb_packet;
	}
	/* Opaque data is not supported */
	return -1;
}

#endif /* RTE_RAW_IOAT */