numam-dpdk/drivers/net/netvsc/hn_rndis.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright (c) 2009-2018 Microsoft Corp.
 * Copyright (c) 2010-2012 Citrix Inc.
 * Copyright (c) 2012 NetApp Inc.
 * All rights reserved.
 */

#include <stdint.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>
#include <unistd.h>

#include <rte_ethdev_driver.h>
#include <rte_ethdev.h>
#include <rte_string_fns.h>
#include <rte_memzone.h>
#include <rte_malloc.h>
#include <rte_atomic.h>
#include <rte_branch_prediction.h>
#include <rte_ether.h>
#include <rte_common.h>
#include <rte_errno.h>
#include <rte_cycles.h>
#include <rte_memory.h>
#include <rte_eal.h>
#include <rte_dev.h>
#include <rte_bus_vmbus.h>

#include "hn_logs.h"
#include "hn_var.h"
#include "hn_nvs.h"
#include "hn_rndis.h"
#include "ndis.h"

#define HN_RNDIS_XFER_SIZE		0x4000

#define HN_NDIS_TXCSUM_CAP_IP4		\
	(NDIS_TXCSUM_CAP_IP4 | NDIS_TXCSUM_CAP_IP4OPT)
#define HN_NDIS_TXCSUM_CAP_TCP4		\
	(NDIS_TXCSUM_CAP_TCP4 | NDIS_TXCSUM_CAP_TCP4OPT)
#define HN_NDIS_TXCSUM_CAP_TCP6		\
	(NDIS_TXCSUM_CAP_TCP6 | NDIS_TXCSUM_CAP_TCP6OPT | \
	 NDIS_TXCSUM_CAP_IP6EXT)
#define HN_NDIS_TXCSUM_CAP_UDP6		\
	(NDIS_TXCSUM_CAP_UDP6 | NDIS_TXCSUM_CAP_IP6EXT)
#define HN_NDIS_LSOV2_CAP_IP6		\
	(NDIS_LSOV2_CAP_IP6EXT | NDIS_LSOV2_CAP_TCP6OPT)

/* Get unique request id */
static inline uint32_t
hn_rndis_rid(struct hn_data *hv)
{
	uint32_t rid;

	do {
		rid = rte_atomic32_add_return(&hv->rndis_req_id, 1);
	} while (rid == 0);

	return rid;
}

static void *hn_rndis_alloc(size_t size)
{
	return rte_zmalloc("RNDIS", size, PAGE_SIZE);
}

#ifdef RTE_LIBRTE_NETVSC_DEBUG_DUMP
void hn_rndis_dump(const void *buf)
{
	const union {
		struct rndis_msghdr hdr;
		struct rndis_packet_msg pkt;
		struct rndis_init_req init_request;
		struct rndis_init_comp init_complete;
		struct rndis_halt_req halt;
		struct rndis_query_req query_request;
		struct rndis_query_comp query_complete;
		struct rndis_set_req set_request;
		struct rndis_set_comp set_complete;
		struct rndis_reset_req reset_request;
		struct rndis_reset_comp reset_complete;
		struct rndis_keepalive_req keepalive_request;
		struct rndis_keepalive_comp keepalive_complete;
		struct rndis_status_msg indicate_status;
	} *rndis_msg = buf;

	switch (rndis_msg->hdr.type) {
	case RNDIS_PACKET_MSG: {
		const struct rndis_pktinfo *ppi;
		unsigned int ppi_len;

		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS_MSG_PACKET (len %u, data %u:%u, # oob %u %u:%u, pkt %u:%u)\n",
			    rndis_msg->pkt.len,
			    rndis_msg->pkt.dataoffset,
			    rndis_msg->pkt.datalen,
			    rndis_msg->pkt.oobdataelements,
			    rndis_msg->pkt.oobdataoffset,
			    rndis_msg->pkt.oobdatalen,
			    rndis_msg->pkt.pktinfooffset,
			    rndis_msg->pkt.pktinfolen);

		ppi = (const struct rndis_pktinfo *)
			((const char *)buf
			 + RNDIS_PACKET_MSG_OFFSET_ABS(rndis_msg->pkt.pktinfooffset));

		ppi_len = rndis_msg->pkt.pktinfolen;
		while (ppi_len > 0) {
			const void *ppi_data;

			ppi_data = ppi->data;

			rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
				"    PPI (size %u, type %u, offs %u data %#x)\n",
				ppi->size, ppi->type, ppi->offset,
				*(const uint32_t *)ppi_data);
			if (ppi->size == 0)
				break;
			ppi_len -= ppi->size;
			ppi = (const struct rndis_pktinfo *)
				((const char *)ppi + ppi->size);
		}
		break;
	}
	case RNDIS_INITIALIZE_MSG:
		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS_MSG_INIT (len %u id %#x, ver %u.%u max xfer %u)\n",
			    rndis_msg->init_request.len,
			    rndis_msg->init_request.rid,
			    rndis_msg->init_request.ver_major,
			    rndis_msg->init_request.ver_minor,
			    rndis_msg->init_request.max_xfersz);
		break;

	case RNDIS_INITIALIZE_CMPLT:
		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS_MSG_INIT_C (len %u, id %#x, status 0x%x, vers %u.%u, "
			    "flags %d, max xfer %u, max pkts %u, aligned %u)\n",
			    rndis_msg->init_complete.len,
			    rndis_msg->init_complete.rid,
			    rndis_msg->init_complete.status,
			    rndis_msg->init_complete.ver_major,
			    rndis_msg->init_complete.ver_minor,
			    rndis_msg->init_complete.devflags,
			    rndis_msg->init_complete.pktmaxsz,
			    rndis_msg->init_complete.pktmaxcnt,
			    rndis_msg->init_complete.align);
		break;

	case RNDIS_HALT_MSG:
		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS_HALT (len %u id %#x)\n",
			    rndis_msg->halt.len, rndis_msg->halt.rid);
		break;

	case RNDIS_QUERY_MSG:
		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS_QUERY (len %u, id %#x, oid %#x, info %u:%u)\n",
			    rndis_msg->query_request.len,
			    rndis_msg->query_request.rid,
			    rndis_msg->query_request.oid,
			    rndis_msg->query_request.infobuflen,
			    rndis_msg->query_request.infobufoffset);
		break;

	case RNDIS_QUERY_CMPLT:
		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS_MSG_QUERY_C (len %u, id %#x, status 0x%x, buf %u:%u)\n",
			    rndis_msg->query_complete.len,
			    rndis_msg->query_complete.rid,
			    rndis_msg->query_complete.status,
			    rndis_msg->query_complete.infobuflen,
			    rndis_msg->query_complete.infobufoffset);
		break;

	case RNDIS_SET_MSG:
		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS_SET (len %u, id %#x, oid %#x, info %u:%u)\n",
			    rndis_msg->set_request.len,
			    rndis_msg->set_request.rid,
			    rndis_msg->set_request.oid,
			    rndis_msg->set_request.infobuflen,
			    rndis_msg->set_request.infobufoffset);
		break;

	case RNDIS_SET_CMPLT:
		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS_MSG_SET_C (len %u, id 0x%x, status 0x%x)\n",
			    rndis_msg->set_complete.len,
			    rndis_msg->set_complete.rid,
			    rndis_msg->set_complete.status);
		break;

	case RNDIS_INDICATE_STATUS_MSG:
		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS_MSG_INDICATE (len %u, status %#x, buf len %u, buf offset %u)\n",
			    rndis_msg->indicate_status.len,
			    rndis_msg->indicate_status.status,
			    rndis_msg->indicate_status.stbuflen,
			    rndis_msg->indicate_status.stbufoffset);
		break;

	case RNDIS_RESET_MSG:
		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS_RESET (len %u, id %#x)\n",
			    rndis_msg->reset_request.len,
			    rndis_msg->reset_request.rid);
		break;

	case RNDIS_RESET_CMPLT:
		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS_RESET_C (len %u, status %#x address %#x)\n",
			    rndis_msg->reset_complete.len,
			    rndis_msg->reset_complete.status,
			    rndis_msg->reset_complete.adrreset);
		break;

	case RNDIS_KEEPALIVE_MSG:
		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS_KEEPALIVE (len %u, id %#x)\n",
			    rndis_msg->keepalive_request.len,
			    rndis_msg->keepalive_request.rid);
		break;

	case RNDIS_KEEPALIVE_CMPLT:
		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS_KEEPALIVE_C (len %u, id %#x address %#x)\n",
			    rndis_msg->keepalive_complete.len,
			    rndis_msg->keepalive_complete.rid,
			    rndis_msg->keepalive_complete.status);
		break;

	default:
		rte_log(RTE_LOG_DEBUG, hn_logtype_driver,
			    "RNDIS type %#x len %u\n",
			    rndis_msg->hdr.type,
			    rndis_msg->hdr.len);
		break;
	}
}
#endif

static int hn_nvs_send_rndis_ctrl(struct vmbus_channel *chan,
				  const void *req, uint32_t reqlen)

{
	struct hn_nvs_rndis nvs_rndis = {
		.type = NVS_TYPE_RNDIS,
		.rndis_mtype = NVS_RNDIS_MTYPE_CTRL,
		.chim_idx = NVS_CHIM_IDX_INVALID,
		.chim_sz = 0
	};
	struct vmbus_gpa sg;
	rte_iova_t addr;

	addr = rte_malloc_virt2iova(req);
	if (unlikely(addr == RTE_BAD_IOVA)) {
		PMD_DRV_LOG(ERR, "RNDIS send request can not get iova");
		return -EINVAL;
	}

	if (unlikely(reqlen > PAGE_SIZE)) {
		PMD_DRV_LOG(ERR, "RNDIS request %u greater than page size",
			    reqlen);
		return -EINVAL;
	}

	sg.page = addr / PAGE_SIZE;
	sg.ofs  = addr & PAGE_MASK;
	sg.len  = reqlen;

	if (sg.ofs + reqlen >  PAGE_SIZE) {
		PMD_DRV_LOG(ERR, "RNDIS request crosses page bounary");
		return -EINVAL;
	}

	hn_rndis_dump(req);

	return hn_nvs_send_sglist(chan, &sg, 1,
				  &nvs_rndis, sizeof(nvs_rndis), 0U, NULL);
}

void hn_rndis_link_status(struct rte_eth_dev *dev, const void *msg)
{
	const struct rndis_status_msg *indicate = msg;

	hn_rndis_dump(msg);

	PMD_DRV_LOG(DEBUG, "link status %#x", indicate->status);

	switch (indicate->status) {
	case RNDIS_STATUS_NETWORK_CHANGE:
	case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG:
		/* ignore not in DPDK API */
		break;

	case RNDIS_STATUS_LINK_SPEED_CHANGE:
	case RNDIS_STATUS_MEDIA_CONNECT:
	case RNDIS_STATUS_MEDIA_DISCONNECT:
		if (dev->data->dev_conf.intr_conf.lsc &&
		    hn_dev_link_update(dev, 0) == 0)
			_rte_eth_dev_callback_process(dev,
						      RTE_ETH_EVENT_INTR_LSC,
						      NULL);
		break;
	default:
		PMD_DRV_LOG(NOTICE, "unknown RNDIS indication: %#x",
			    indicate->status);
	}
}

/* Callback from hn_process_events when response is visible */
void hn_rndis_receive_response(struct hn_data *hv,
			       const void *data, uint32_t len)
{
	const struct rndis_init_comp *hdr = data;

	hn_rndis_dump(data);

	if (len < sizeof(3 * sizeof(uint32_t))) {
		PMD_DRV_LOG(ERR,
			    "missing RNDIS header %u", len);
		return;
	}

	if (len < hdr->len) {
		PMD_DRV_LOG(ERR,
			    "truncated RNDIS response %u", len);
		return;
	}

	if  (len > sizeof(hv->rndis_resp)) {
		PMD_DRV_LOG(NOTICE,
			    "RNDIS response exceeds buffer");
		len = sizeof(hv->rndis_resp);
	}

	if (hdr->rid == 0) {
		PMD_DRV_LOG(NOTICE,
			    "RNDIS response id zero!");
	}

	memcpy(hv->rndis_resp, data, len);

	/* make sure response copied before update */
	rte_smp_wmb();

	if (rte_atomic32_cmpset(&hv->rndis_pending, hdr->rid, 0) == 0) {
		PMD_DRV_LOG(ERR,
			    "received id %#x pending id %#x",
			    hdr->rid, (uint32_t)hv->rndis_pending);
	}
}

/* Do request/response transaction */
static int hn_rndis_exec1(struct hn_data *hv,
			  const void *req, uint32_t reqlen,
			  void *comp, uint32_t comp_len)
{
	const struct rndis_halt_req *hdr = req;
	uint32_t rid = hdr->rid;
	struct vmbus_channel *chan = hn_primary_chan(hv);
	int error;

	if (comp_len > sizeof(hv->rndis_resp)) {
		PMD_DRV_LOG(ERR,
			    "Expected completion size %u exceeds buffer %zu",
			    comp_len, sizeof(hv->rndis_resp));
		return -EIO;
	}

	if (comp != NULL &&
	    rte_atomic32_cmpset(&hv->rndis_pending, 0, rid) == 0) {
		PMD_DRV_LOG(ERR,
			    "Request already pending");
		return -EBUSY;
	}

	error = hn_nvs_send_rndis_ctrl(chan, req, reqlen);
	if (error) {
		PMD_DRV_LOG(ERR, "RNDIS ctrl send failed: %d", error);
		return error;
	}

	if (comp) {
		/* Poll primary channel until response received */
		while (hv->rndis_pending == rid)
			hn_process_events(hv, 0, 1);

		memcpy(comp, hv->rndis_resp, comp_len);
	}

	return 0;
}

/* Do transaction and validate response */
static int hn_rndis_execute(struct hn_data *hv, uint32_t rid,
			    const void *req, uint32_t reqlen,
			    void *comp, uint32_t comp_len, uint32_t comp_type)
{
	const struct rndis_comp_hdr *hdr = comp;
	int ret;

	memset(comp, 0, comp_len);

	ret = hn_rndis_exec1(hv, req, reqlen, comp, comp_len);
	if (ret < 0)
		return ret;
	/*
	 * Check this RNDIS complete message.
	 */
	if (unlikely(hdr->type != comp_type)) {
		PMD_DRV_LOG(ERR,
			    "unexpected RNDIS response complete %#x expect %#x",
			    hdr->type, comp_type);

		return -ENXIO;
	}
	if (unlikely(hdr->rid != rid)) {
		PMD_DRV_LOG(ERR,
			    "RNDIS comp rid mismatch %#x, expect %#x",
			    hdr->rid, rid);
		return -EINVAL;
	}

	/* All pass! */
	return 0;
}

static int
hn_rndis_query(struct hn_data *hv, uint32_t oid,
	       const void *idata, uint32_t idlen,
	       void *odata, uint32_t odlen)
{
	struct rndis_query_req *req;
	struct rndis_query_comp *comp;
	uint32_t reqlen, comp_len;
	int error = -EIO;
	unsigned int ofs;
	uint32_t rid;

	reqlen = sizeof(*req) + idlen;
	req = hn_rndis_alloc(reqlen);
	if (req == NULL)
		return -ENOMEM;

	comp_len = sizeof(*comp) + odlen;
	comp = rte_zmalloc("QUERY", comp_len, PAGE_SIZE);
	if (!comp) {
		error = -ENOMEM;
		goto done;
	}
	comp->status = RNDIS_STATUS_PENDING;

	rid = hn_rndis_rid(hv);

	req->type = RNDIS_QUERY_MSG;
	req->len = reqlen;
	req->rid = rid;
	req->oid = oid;
	req->infobufoffset = RNDIS_QUERY_REQ_INFOBUFOFFSET;
	req->infobuflen = idlen;

	/* Input data immediately follows RNDIS query. */
	memcpy(req + 1, idata, idlen);

	error = hn_rndis_execute(hv, rid, req, reqlen,
				 comp, comp_len, RNDIS_QUERY_CMPLT);

	if (error)
		goto done;

	if (comp->status != RNDIS_STATUS_SUCCESS) {
		PMD_DRV_LOG(ERR, "RNDIS query 0x%08x failed: status 0x%08x",
			    oid, comp->status);
		error = -EINVAL;
		goto done;
	}

	if (comp->infobuflen == 0 || comp->infobufoffset == 0) {
		/* No output data! */
		PMD_DRV_LOG(ERR, "RNDIS query 0x%08x, no data", oid);
		error = 0;
		goto done;
	}

	/*
	 * Check output data length and offset.
	 */
	/* ofs is the offset from the beginning of comp. */
	ofs = RNDIS_QUERY_COMP_INFOBUFOFFSET_ABS(comp->infobufoffset);
	if (ofs < sizeof(*comp) || ofs + comp->infobuflen > comp_len) {
		PMD_DRV_LOG(ERR, "RNDIS query invalid comp ib off/len, %u/%u",
			    comp->infobufoffset, comp->infobuflen);
		error = -EINVAL;
		goto done;
	}

	/* Save output data. */
	if (comp->infobuflen < odlen)
		odlen = comp->infobuflen;

	/* ofs is the offset from the beginning of comp. */
	memcpy(odata, (const char *)comp + ofs, odlen);

	error = 0;
done:
	rte_free(comp);
	rte_free(req);
	return error;
}

static int
hn_rndis_halt(struct hn_data *hv)
{
	struct rndis_halt_req *halt;

	halt = hn_rndis_alloc(sizeof(*halt));
	if (halt == NULL)
		return -ENOMEM;

	halt->type = RNDIS_HALT_MSG;
	halt->len = sizeof(*halt);
	halt->rid = hn_rndis_rid(hv);

	/* No RNDIS completion; rely on NVS message send completion */
	hn_rndis_exec1(hv, halt, sizeof(*halt), NULL, 0);

	rte_free(halt);

	PMD_INIT_LOG(DEBUG, "RNDIS halt done");
	return 0;
}

static int
hn_rndis_query_hwcaps(struct hn_data *hv, struct ndis_offload *caps)
{
	struct ndis_offload in;
	uint32_t caps_len, size;
	int error;

	memset(caps, 0, sizeof(*caps));
	memset(&in, 0, sizeof(in));
	in.ndis_hdr.ndis_type = NDIS_OBJTYPE_OFFLOAD;

	if (hv->ndis_ver >= NDIS_VERSION_6_30) {
		in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_3;
		size = NDIS_OFFLOAD_SIZE;
	} else if (hv->ndis_ver >= NDIS_VERSION_6_1) {
		in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_2;
		size = NDIS_OFFLOAD_SIZE_6_1;
	} else {
		in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_1;
		size = NDIS_OFFLOAD_SIZE_6_0;
	}
	in.ndis_hdr.ndis_size = size;

	caps_len = NDIS_OFFLOAD_SIZE;
	error = hn_rndis_query(hv, OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES,
			       &in, size, caps, caps_len);
	if (error)
		return error;

	/* Preliminary verification. */
	if (caps->ndis_hdr.ndis_type != NDIS_OBJTYPE_OFFLOAD) {
		PMD_DRV_LOG(NOTICE, "invalid NDIS objtype 0x%02x",
			    caps->ndis_hdr.ndis_type);
		return -EINVAL;
	}
	if (caps->ndis_hdr.ndis_rev < NDIS_OFFLOAD_REV_1) {
		PMD_DRV_LOG(NOTICE, "invalid NDIS objrev 0x%02x",
			    caps->ndis_hdr.ndis_rev);
		return -EINVAL;
	}
	if (caps->ndis_hdr.ndis_size > caps_len) {
		PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u, data size %u",
			    caps->ndis_hdr.ndis_size, caps_len);
		return -EINVAL;
	} else if (caps->ndis_hdr.ndis_size < NDIS_OFFLOAD_SIZE_6_0) {
		PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u",
			    caps->ndis_hdr.ndis_size);
		return -EINVAL;
	}

	return 0;
}

int
hn_rndis_query_rsscaps(struct hn_data *hv,
		       unsigned int *rxr_cnt0)
{
	struct ndis_rss_caps in, caps;
	unsigned int indsz, rxr_cnt;
	uint32_t caps_len;
	int error;

	*rxr_cnt0 = 0;

	if (hv->ndis_ver < NDIS_VERSION_6_20) {
		PMD_DRV_LOG(DEBUG, "RSS not supported on this host");
		return -EOPNOTSUPP;
	}

	memset(&in, 0, sizeof(in));
	in.ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_CAPS;
	in.ndis_hdr.ndis_rev = NDIS_RSS_CAPS_REV_2;
	in.ndis_hdr.ndis_size = NDIS_RSS_CAPS_SIZE;

	caps_len = NDIS_RSS_CAPS_SIZE;
	error = hn_rndis_query(hv, OID_GEN_RECEIVE_SCALE_CAPABILITIES,
			       &in, NDIS_RSS_CAPS_SIZE,
			       &caps, caps_len);
	if (error)
		return error;

	PMD_INIT_LOG(DEBUG, "RX rings %u indirect %u caps %#x",
		     caps.ndis_nrxr, caps.ndis_nind, caps.ndis_caps);
	/*
	 * Preliminary verification.
	 */
	if (caps.ndis_hdr.ndis_type != NDIS_OBJTYPE_RSS_CAPS) {
		PMD_DRV_LOG(ERR, "invalid NDIS objtype 0x%02x",
			    caps.ndis_hdr.ndis_type);
		return -EINVAL;
	}
	if (caps.ndis_hdr.ndis_rev < NDIS_RSS_CAPS_REV_1) {
		PMD_DRV_LOG(ERR, "invalid NDIS objrev 0x%02x",
			    caps.ndis_hdr.ndis_rev);
		return -EINVAL;
	}
	if (caps.ndis_hdr.ndis_size > caps_len) {
		PMD_DRV_LOG(ERR,
			    "invalid NDIS objsize %u, data size %u",
			    caps.ndis_hdr.ndis_size, caps_len);
		return -EINVAL;
	} else if (caps.ndis_hdr.ndis_size < NDIS_RSS_CAPS_SIZE_6_0) {
		PMD_DRV_LOG(ERR, "invalid NDIS objsize %u",
			    caps.ndis_hdr.ndis_size);
		return -EINVAL;
	}

	/*
	 * Save information for later RSS configuration.
	 */
	if (caps.ndis_nrxr == 0) {
		PMD_DRV_LOG(ERR, "0 RX rings!?");
		return -EINVAL;
	}
	rxr_cnt = caps.ndis_nrxr;

	if (caps.ndis_hdr.ndis_size == NDIS_RSS_CAPS_SIZE &&
	    caps.ndis_hdr.ndis_rev >= NDIS_RSS_CAPS_REV_2) {
		if (caps.ndis_nind > NDIS_HASH_INDCNT) {
			PMD_DRV_LOG(ERR,
				    "too many RSS indirect table entries %u",
				    caps.ndis_nind);
			return -EOPNOTSUPP;
		}
		if (!rte_is_power_of_2(caps.ndis_nind)) {
			PMD_DRV_LOG(ERR,
				    "RSS indirect table size is not power-of-2 %u",
				    caps.ndis_nind);
		}

		indsz = caps.ndis_nind;
	} else {
		indsz = NDIS_HASH_INDCNT;
	}

	if (indsz < rxr_cnt) {
		PMD_DRV_LOG(NOTICE,
			    "# of RX rings (%d) > RSS indirect table size %d",
			    rxr_cnt, indsz);
		rxr_cnt = indsz;
	}

	hv->rss_offloads = 0;
	if (caps.ndis_caps & NDIS_RSS_CAP_IPV4)
		hv->rss_offloads |= ETH_RSS_IPV4
			| ETH_RSS_NONFRAG_IPV4_TCP
			| ETH_RSS_NONFRAG_IPV4_UDP;
	if (caps.ndis_caps & NDIS_RSS_CAP_IPV6)
		hv->rss_offloads |= ETH_RSS_IPV6
			| ETH_RSS_NONFRAG_IPV6_TCP;
	if (caps.ndis_caps & NDIS_RSS_CAP_IPV6_EX)
		hv->rss_offloads |= ETH_RSS_IPV6_EX
			| ETH_RSS_IPV6_TCP_EX;

	/* Commit! */
	*rxr_cnt0 = rxr_cnt;

	return 0;
}

static int
hn_rndis_set(struct hn_data *hv, uint32_t oid, const void *data, uint32_t dlen)
{
	struct rndis_set_req *req;
	struct rndis_set_comp comp;
	uint32_t reqlen, comp_len;
	uint32_t rid;
	int error;

	reqlen = sizeof(*req) + dlen;
	req = rte_zmalloc("RNDIS_SET", reqlen, PAGE_SIZE);
	if (!req)
		return -ENOMEM;

	rid = hn_rndis_rid(hv);
	req->type = RNDIS_SET_MSG;
	req->len = reqlen;
	req->rid = rid;
	req->oid = oid;
	req->infobuflen = dlen;
	req->infobufoffset = RNDIS_SET_REQ_INFOBUFOFFSET;

	/* Data immediately follows RNDIS set. */
	memcpy(req + 1, data, dlen);

	comp_len = sizeof(comp);
	error = hn_rndis_execute(hv, rid, req, reqlen,
				 &comp, comp_len,
				 RNDIS_SET_CMPLT);
	if (error) {
		PMD_DRV_LOG(ERR, "exec RNDIS set %#" PRIx32 " failed",
			    oid);
		error = EIO;
		goto done;
	}

	if (comp.status != RNDIS_STATUS_SUCCESS) {
		PMD_DRV_LOG(ERR,
			    "RNDIS set %#" PRIx32 " failed: status %#" PRIx32,
			    oid, comp.status);
		error = EIO;
		goto done;
	}

done:
	rte_free(req);
	return error;
}

int hn_rndis_conf_offload(struct hn_data *hv,
			  uint64_t tx_offloads, uint64_t rx_offloads)
{
	struct ndis_offload_params params;
	struct ndis_offload hwcaps;
	int error;

	error = hn_rndis_query_hwcaps(hv, &hwcaps);
	if (error) {
		PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error);
		return error;
	}

	/* NOTE: 0 means "no change" */
	memset(&params, 0, sizeof(params));

	params.ndis_hdr.ndis_type = NDIS_OBJTYPE_DEFAULT;
	if (hv->ndis_ver < NDIS_VERSION_6_30) {
		params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_2;
		params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE_6_1;
	} else {
		params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_3;
		params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE;
	}

	if (tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) {
		if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_TCP4)
			params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TX;
		else
			goto unsupported;

		if (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_TCP6)
			params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TX;
		else
			goto unsupported;
	}

	if (rx_offloads & DEV_RX_OFFLOAD_TCP_CKSUM) {
		if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4)
		    == NDIS_RXCSUM_CAP_TCP4)
			params.ndis_tcp4csum |= NDIS_OFFLOAD_PARAM_RX;
		else
			goto unsupported;

		if ((hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)
		    == NDIS_RXCSUM_CAP_TCP6)
			params.ndis_tcp6csum |= NDIS_OFFLOAD_PARAM_RX;
		else
			goto unsupported;
	}

	if (tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) {
		if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4)
			params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TX;
		else
			goto unsupported;

		if ((hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6)
		    == NDIS_TXCSUM_CAP_UDP6)
			params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TX;
		else
			goto unsupported;
	}

	if (rx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) {
		if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4)
			params.ndis_udp4csum |= NDIS_OFFLOAD_PARAM_RX;
		else
			goto unsupported;

		if (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)
			params.ndis_udp6csum |= NDIS_OFFLOAD_PARAM_RX;
		else
			goto unsupported;
	}

	if (tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) {
		if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_IP4)
		    == NDIS_TXCSUM_CAP_IP4)
			params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TX;
		else
			goto unsupported;
	}
	if (rx_offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) {
		if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
			params.ndis_ip4csum |= NDIS_OFFLOAD_PARAM_RX;
		else
			goto unsupported;
	}

	if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO) {
		if (hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023)
			params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_ON;
		else
			goto unsupported;

		if ((hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6)
		    == HN_NDIS_LSOV2_CAP_IP6)
			params.ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_ON;
		else
			goto unsupported;
	}

	error = hn_rndis_set(hv, OID_TCP_OFFLOAD_PARAMETERS, &params,
			     params.ndis_hdr.ndis_size);
	if (error) {
		PMD_DRV_LOG(ERR, "offload config failed");
		return error;
	}

	return 0;
 unsupported:
	PMD_DRV_LOG(NOTICE,
		    "offload tx:%" PRIx64 " rx:%" PRIx64 " not supported by this version",
		    tx_offloads, rx_offloads);
	return -EINVAL;
}

int hn_rndis_get_offload(struct hn_data *hv,
			 struct rte_eth_dev_info *dev_info)
{
	struct ndis_offload hwcaps;
	int error;

	memset(&hwcaps, 0, sizeof(hwcaps));

	error = hn_rndis_query_hwcaps(hv, &hwcaps);
	if (error) {
		PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error);
		return error;
	}

	dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS |
				    DEV_TX_OFFLOAD_VLAN_INSERT;

	if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_IP4)
	    == HN_NDIS_TXCSUM_CAP_IP4)
		dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM;

	if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_TCP4)
	    == HN_NDIS_TXCSUM_CAP_TCP4 &&
	    (hwcaps.ndis_csum.ndis_ip6_txcsum & HN_NDIS_TXCSUM_CAP_TCP6)
	    == HN_NDIS_TXCSUM_CAP_TCP6)
		dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_CKSUM;

	if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) &&
	    (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6))
		dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_UDP_CKSUM;

	if ((hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) &&
	    (hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6)
	    == HN_NDIS_LSOV2_CAP_IP6)
		dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;

	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
				    DEV_RX_OFFLOAD_RSS_HASH;

	if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
		dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM;

	if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) &&
	    (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6))
		dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_CKSUM;

	if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) &&
	    (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6))
		dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_UDP_CKSUM;

	return 0;
}

uint32_t
hn_rndis_get_ptypes(struct hn_data *hv)
{
	struct ndis_offload hwcaps;
	uint32_t ptypes;
	int error;

	memset(&hwcaps, 0, sizeof(hwcaps));

	error = hn_rndis_query_hwcaps(hv, &hwcaps);
	if (error) {
		PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error);
		return RTE_PTYPE_L2_ETHER;
	}

	ptypes = RTE_PTYPE_L2_ETHER;

	if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4)
		ptypes |= RTE_PTYPE_L3_IPV4;

	if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) ||
	    (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6))
		ptypes |= RTE_PTYPE_L4_TCP;

	if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) ||
	    (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6))
		ptypes |= RTE_PTYPE_L4_UDP;

	return ptypes;
}

int
hn_rndis_set_rxfilter(struct hn_data *hv, uint32_t filter)
{
	int error;

	error = hn_rndis_set(hv, OID_GEN_CURRENT_PACKET_FILTER,
			     &filter, sizeof(filter));
	if (error) {
		PMD_DRV_LOG(ERR, "set RX filter %#" PRIx32 " failed: %d",
			    filter, error);
	} else {
		PMD_DRV_LOG(DEBUG, "set RX filter %#" PRIx32 " done", filter);
	}

	return error;
}

int hn_rndis_conf_rss(struct hn_data *hv, uint32_t flags)
{
	struct ndis_rssprm_toeplitz rssp;
	struct ndis_rss_params *prm = &rssp.rss_params;
	unsigned int i;
	int error;

	memset(&rssp, 0, sizeof(rssp));

	prm->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS;
	prm->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2;
	prm->ndis_hdr.ndis_size = sizeof(*prm);
	prm->ndis_flags = flags;
	prm->ndis_hash = hv->rss_hash;
	prm->ndis_indsize = sizeof(rssp.rss_ind[0]) * NDIS_HASH_INDCNT;
	prm->ndis_indoffset = offsetof(struct ndis_rssprm_toeplitz, rss_ind[0]);
	prm->ndis_keysize = NDIS_HASH_KEYSIZE_TOEPLITZ;
	prm->ndis_keyoffset = offsetof(struct ndis_rssprm_toeplitz, rss_key[0]);

	for (i = 0; i < NDIS_HASH_INDCNT; i++)
		rssp.rss_ind[i] = hv->rss_ind[i];

	/* Set hask key values */
	memcpy(&rssp.rss_key, hv->rss_key, NDIS_HASH_KEYSIZE_TOEPLITZ);

	error = hn_rndis_set(hv, OID_GEN_RECEIVE_SCALE_PARAMETERS,
			     &rssp, sizeof(rssp));
	if (error != 0) {
		PMD_DRV_LOG(ERR,
			    "RSS config num queues=%u failed: %d",
			    hv->num_queues, error);
	}
	return error;
}

static int hn_rndis_init(struct hn_data *hv)
{
	struct rndis_init_req *req;
	struct rndis_init_comp comp;
	uint32_t comp_len, rid;
	int error;

	req = hn_rndis_alloc(sizeof(*req));
	if (!req) {
		PMD_DRV_LOG(ERR, "no memory for RNDIS init");
		return -ENXIO;
	}

	rid = hn_rndis_rid(hv);
	req->type = RNDIS_INITIALIZE_MSG;
	req->len = sizeof(*req);
	req->rid = rid;
	req->ver_major = RNDIS_VERSION_MAJOR;
	req->ver_minor = RNDIS_VERSION_MINOR;
	req->max_xfersz = HN_RNDIS_XFER_SIZE;

	comp_len = RNDIS_INIT_COMP_SIZE_MIN;
	error = hn_rndis_execute(hv, rid, req, sizeof(*req),
				 &comp, comp_len,
				 RNDIS_INITIALIZE_CMPLT);
	if (error)
		goto done;

	if (comp.status != RNDIS_STATUS_SUCCESS) {
		PMD_DRV_LOG(ERR, "RNDIS init failed: status 0x%08x",
			    comp.status);
		error = -EIO;
		goto done;
	}

	hv->rndis_agg_size = comp.pktmaxsz;
	hv->rndis_agg_pkts = comp.pktmaxcnt;
	hv->rndis_agg_align = 1U << comp.align;

	if (hv->rndis_agg_align < sizeof(uint32_t)) {
		/*
		 * The RNDIS packet message encap assumes that the RNDIS
		 * packet message is at least 4 bytes aligned.  Fix up the
		 * alignment here, if the remote side sets the alignment
		 * too low.
		 */
		PMD_DRV_LOG(NOTICE,
			    "fixup RNDIS aggpkt align: %u -> %zu",
			    hv->rndis_agg_align, sizeof(uint32_t));
		hv->rndis_agg_align = sizeof(uint32_t);
	}

	PMD_INIT_LOG(INFO,
		     "RNDIS ver %u.%u, aggpkt size %u, aggpkt cnt %u, aggpkt align %u",
		     comp.ver_major, comp.ver_minor,
		     hv->rndis_agg_size, hv->rndis_agg_pkts,
		     hv->rndis_agg_align);
	error = 0;
done:
	rte_free(req);
	return error;
}

int
hn_rndis_get_eaddr(struct hn_data *hv, uint8_t *eaddr)
{
	uint32_t eaddr_len;
	int error;

	eaddr_len = RTE_ETHER_ADDR_LEN;
	error = hn_rndis_query(hv, OID_802_3_PERMANENT_ADDRESS, NULL, 0,
			       eaddr, eaddr_len);
	if (error)
		return error;

	PMD_DRV_LOG(INFO, "MAC address %02x:%02x:%02x:%02x:%02x:%02x",
		    eaddr[0], eaddr[1], eaddr[2],
		    eaddr[3], eaddr[4], eaddr[5]);
	return 0;
}

int
hn_rndis_get_linkstatus(struct hn_data *hv)
{
	return hn_rndis_query(hv, OID_GEN_MEDIA_CONNECT_STATUS, NULL, 0,
			      &hv->link_status, sizeof(uint32_t));
}

int
hn_rndis_get_linkspeed(struct hn_data *hv)
{
	return hn_rndis_query(hv, OID_GEN_LINK_SPEED, NULL, 0,
			      &hv->link_speed, sizeof(uint32_t));
}

int
hn_rndis_attach(struct hn_data *hv)
{
	/* Initialize RNDIS. */
	return hn_rndis_init(hv);
}

void
hn_rndis_detach(struct hn_data *hv)
{
	/* Halt the RNDIS. */
	hn_rndis_halt(hv);
}