numam-dpdk/drivers/net/failsafe/failsafe_ether.c

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright 2017 6WIND S.A.
 * Copyright 2017 Mellanox Technologies, Ltd
 */

#include <unistd.h>

#include <rte_flow.h>
#include <rte_flow_driver.h>
#include <rte_cycles.h>

#include "failsafe_private.h"

/** Print a message out of a flow error. */
static int
fs_flow_complain(struct rte_flow_error *error)
{
	static const char *const errstrlist[] = {
		[RTE_FLOW_ERROR_TYPE_NONE] = "no error",
		[RTE_FLOW_ERROR_TYPE_UNSPECIFIED] = "cause unspecified",
		[RTE_FLOW_ERROR_TYPE_HANDLE] = "flow rule (handle)",
		[RTE_FLOW_ERROR_TYPE_ATTR_GROUP] = "group field",
		[RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY] = "priority field",
		[RTE_FLOW_ERROR_TYPE_ATTR_INGRESS] = "ingress field",
		[RTE_FLOW_ERROR_TYPE_ATTR_EGRESS] = "egress field",
		[RTE_FLOW_ERROR_TYPE_ATTR] = "attributes structure",
		[RTE_FLOW_ERROR_TYPE_ITEM_NUM] = "pattern length",
		[RTE_FLOW_ERROR_TYPE_ITEM] = "specific pattern item",
		[RTE_FLOW_ERROR_TYPE_ACTION_NUM] = "number of actions",
		[RTE_FLOW_ERROR_TYPE_ACTION] = "specific action",
	};
	const char *errstr;
	char buf[32];
	int err = rte_errno;

	if ((unsigned int)error->type >= RTE_DIM(errstrlist) ||
			!errstrlist[error->type])
		errstr = "unknown type";
	else
		errstr = errstrlist[error->type];
	ERROR("Caught error type %d (%s): %s%s\n",
		error->type, errstr,
		error->cause ? (snprintf(buf, sizeof(buf), "cause: %p, ",
				error->cause), buf) : "",
		error->message ? error->message : "(no stated reason)");
	return -err;
}

static int
eth_dev_flow_isolate_set(struct rte_eth_dev *dev,
			 struct sub_device *sdev)
{
	struct rte_flow_error ferror;
	int ret;

	if (!PRIV(dev)->flow_isolated) {
		DEBUG("Flow isolation already disabled");
	} else {
		DEBUG("Enabling flow isolation");
		ret = rte_flow_isolate(PORT_ID(sdev),
				       PRIV(dev)->flow_isolated,
				       &ferror);
		if (ret) {
			fs_flow_complain(&ferror);
			return ret;
		}
	}
	return 0;
}

static int
fs_eth_dev_conf_apply(struct rte_eth_dev *dev,
		struct sub_device *sdev)
{
	struct rte_eth_dev *edev;
	struct rte_vlan_filter_conf *vfc1;
	struct rte_vlan_filter_conf *vfc2;
	struct rte_flow *flow;
	struct rte_flow_error ferror;
	uint32_t i;
	int ret;

	edev = ETH(sdev);
	/* RX queue setup */
	for (i = 0; i < dev->data->nb_rx_queues; i++) {
		struct rxq *rxq;

		rxq = dev->data->rx_queues[i];
		ret = rte_eth_rx_queue_setup(PORT_ID(sdev), i,
				rxq->info.nb_desc, rxq->socket_id,
				&rxq->info.conf, rxq->info.mp);
		if (ret) {
			ERROR("rx_queue_setup failed");
			return ret;
		}
	}
	/* TX queue setup */
	for (i = 0; i < dev->data->nb_tx_queues; i++) {
		struct txq *txq;

		txq = dev->data->tx_queues[i];
		ret = rte_eth_tx_queue_setup(PORT_ID(sdev), i,
				txq->info.nb_desc, txq->socket_id,
				&txq->info.conf);
		if (ret) {
			ERROR("tx_queue_setup failed");
			return ret;
		}
	}
	/* dev_link.link_status */
	if (dev->data->dev_link.link_status !=
	    edev->data->dev_link.link_status) {
		DEBUG("Configuring link_status");
		if (dev->data->dev_link.link_status)
			ret = rte_eth_dev_set_link_up(PORT_ID(sdev));
		else
			ret = rte_eth_dev_set_link_down(PORT_ID(sdev));
		if (ret) {
			ERROR("Failed to apply link_status");
			return ret;
		}
	} else {
		DEBUG("link_status already set");
	}
	/* promiscuous */
	if (dev->data->promiscuous != edev->data->promiscuous) {
		DEBUG("Configuring promiscuous");
		if (dev->data->promiscuous)
			ret = rte_eth_promiscuous_enable(PORT_ID(sdev));
		else
			ret = rte_eth_promiscuous_disable(PORT_ID(sdev));
		if (ret != 0) {
			ERROR("Failed to apply promiscuous mode");
			return ret;
		}
	} else {
		DEBUG("promiscuous already set");
	}
	/* all_multicast */
	if (dev->data->all_multicast != edev->data->all_multicast) {
		DEBUG("Configuring all_multicast");
		if (dev->data->all_multicast)
			ret = rte_eth_allmulticast_enable(PORT_ID(sdev));
		else
			ret = rte_eth_allmulticast_disable(PORT_ID(sdev));
		if (ret != 0) {
			ERROR("Failed to apply allmulticast mode");
			return ret;
		}
	} else {
		DEBUG("all_multicast already set");
	}
	/* MTU */
	if (dev->data->mtu != edev->data->mtu) {
		DEBUG("Configuring MTU");
		ret = rte_eth_dev_set_mtu(PORT_ID(sdev), dev->data->mtu);
		if (ret) {
			ERROR("Failed to apply MTU");
			return ret;
		}
	} else {
		DEBUG("MTU already set");
	}
	/* default MAC */
	DEBUG("Configuring default MAC address");
	ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev),
			&dev->data->mac_addrs[0]);
	if (ret) {
		ERROR("Setting default MAC address failed");
		return ret;
	}
	/* additional MAC */
	if (PRIV(dev)->nb_mac_addr > 1)
		DEBUG("Configure additional MAC address%s",
			(PRIV(dev)->nb_mac_addr > 2 ? "es" : ""));
	for (i = 1; i < PRIV(dev)->nb_mac_addr; i++) {
		struct rte_ether_addr *ea;

		ea = &dev->data->mac_addrs[i];
		ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), ea,
				PRIV(dev)->mac_addr_pool[i]);
		if (ret) {
			char ea_fmt[RTE_ETHER_ADDR_FMT_SIZE];

			rte_ether_format_addr(ea_fmt,
					RTE_ETHER_ADDR_FMT_SIZE, ea);
			ERROR("Adding MAC address %s failed", ea_fmt);
			return ret;
		}
	}
	/*
	 * Propagate multicast MAC addresses to sub-devices,
	 * if non zero number of addresses is set.
	 * The condition is required to avoid breakage of failsafe
	 * for sub-devices which do not support the operation
	 * if the feature is really not used.
	 */
	if (PRIV(dev)->nb_mcast_addr > 0) {
		DEBUG("Configuring multicast MAC addresses");
		ret = rte_eth_dev_set_mc_addr_list(PORT_ID(sdev),
						   PRIV(dev)->mcast_addrs,
						   PRIV(dev)->nb_mcast_addr);
		if (ret) {
			ERROR("Failed to apply multicast MAC addresses");
			return ret;
		}
	}
	/* VLAN filter */
	vfc1 = &dev->data->vlan_filter_conf;
	vfc2 = &edev->data->vlan_filter_conf;
	if (memcmp(vfc1, vfc2, sizeof(struct rte_vlan_filter_conf))) {
		uint64_t vbit;
		uint64_t ids;
		size_t i;
		uint16_t vlan_id;

		DEBUG("Configuring VLAN filter");
		for (i = 0; i < RTE_DIM(vfc1->ids); i++) {
			if (vfc1->ids[i] == 0)
				continue;
			ids = vfc1->ids[i];
			while (ids) {
				vlan_id = 64 * i;
				/* count trailing zeroes */
				vbit = ~ids & (ids - 1);
				/* clear least significant bit set */
				ids ^= (ids ^ (ids - 1)) ^ vbit;
				for (; vbit; vlan_id++)
					vbit >>= 1;
				ret = rte_eth_dev_vlan_filter(
					PORT_ID(sdev), vlan_id, 1);
				if (ret) {
					ERROR("Failed to apply VLAN filter %hu",
						vlan_id);
					return ret;
				}
			}
		}
	} else {
		DEBUG("VLAN filter already set");
	}
	/* rte_flow */
	if (TAILQ_EMPTY(&PRIV(dev)->flow_list)) {
		DEBUG("rte_flow already set");
	} else {
		DEBUG("Resetting rte_flow configuration");
		ret = rte_flow_flush(PORT_ID(sdev), &ferror);
		if (ret) {
			fs_flow_complain(&ferror);
			return ret;
		}
		i = 0;
		rte_errno = 0;
		DEBUG("Configuring rte_flow");
		TAILQ_FOREACH(flow, &PRIV(dev)->flow_list, next) {
			DEBUG("Creating flow #%" PRIu32, i++);
			flow->flows[SUB_ID(sdev)] =
				rte_flow_create(PORT_ID(sdev),
						flow->rule.attr,
						flow->rule.pattern,
						flow->rule.actions,
						&ferror);
			ret = rte_errno;
			if (ret)
				break;
		}
		if (ret) {
			fs_flow_complain(&ferror);
			return ret;
		}
	}
	return 0;
}

static void
fs_dev_remove(struct sub_device *sdev)
{
	int ret;

	if (sdev == NULL)
		return;
	switch (sdev->state) {
	case DEV_STARTED:
		failsafe_rx_intr_uninstall_subdevice(sdev);
		ret = rte_eth_dev_stop(PORT_ID(sdev));
		if (ret < 0)
			ERROR("Failed to stop sub-device %u", SUB_ID(sdev));
		sdev->state = DEV_ACTIVE;
		/* fallthrough */
	case DEV_ACTIVE:
		failsafe_eth_dev_unregister_callbacks(sdev);
		ret = rte_eth_dev_close(PORT_ID(sdev));
		if (ret < 0) {
			ERROR("Port close failed for sub-device %u",
			      PORT_ID(sdev));
		}
		sdev->state = DEV_PROBED;
		/* fallthrough */
	case DEV_PROBED:
		ret = rte_dev_remove(sdev->dev);
		if (ret < 0) {
			ERROR("Bus detach failed for sub_device %u",
			      SUB_ID(sdev));
		} else {
			rte_eth_dev_release_port(ETH(sdev));
		}
		sdev->state = DEV_PARSED;
		/* fallthrough */
	case DEV_PARSED:
	case DEV_UNDEFINED:
		sdev->state = DEV_UNDEFINED;
		sdev->sdev_port_id = RTE_MAX_ETHPORTS;
		/* the end */
		break;
	}
	sdev->remove = 0;
	failsafe_hotplug_alarm_install(fs_dev(sdev));
}

static void
fs_dev_stats_save(struct sub_device *sdev)
{
	struct rte_eth_stats stats;
	int err;

	/* Attempt to read current stats. */
	err = rte_eth_stats_get(PORT_ID(sdev), &stats);
	if (err) {
		uint64_t timestamp = sdev->stats_snapshot.timestamp;

		WARN("Could not access latest statistics from sub-device %d.",
			 SUB_ID(sdev));
		if (timestamp != 0)
			WARN("Using latest snapshot taken before %"PRIu64" seconds.",
				 (rte_rdtsc() - timestamp) / rte_get_tsc_hz());
	}
	failsafe_stats_increment
		(&PRIV(fs_dev(sdev))->stats_accumulator,
		err ? &sdev->stats_snapshot.stats : &stats);
	memset(&sdev->stats_snapshot, 0, sizeof(sdev->stats_snapshot));
}

static inline int
fs_rxtx_clean(struct sub_device *sdev)
{
	uint16_t i;

	for (i = 0; i < ETH(sdev)->data->nb_rx_queues; i++)
		if (FS_ATOMIC_RX(sdev, i))
			return 0;
	for (i = 0; i < ETH(sdev)->data->nb_tx_queues; i++)
		if (FS_ATOMIC_TX(sdev, i))
			return 0;
	return 1;
}

void
failsafe_eth_dev_unregister_callbacks(struct sub_device *sdev)
{
	int ret;

	if (sdev == NULL)
		return;
	if (sdev->rmv_callback) {
		ret = rte_eth_dev_callback_unregister(PORT_ID(sdev),
						RTE_ETH_EVENT_INTR_RMV,
						failsafe_eth_rmv_event_callback,
						sdev);
		if (ret)
			WARN("Failed to unregister RMV callback for sub_device"
			     " %d", SUB_ID(sdev));
		sdev->rmv_callback = 0;
	}
	if (sdev->lsc_callback) {
		ret = rte_eth_dev_callback_unregister(PORT_ID(sdev),
						RTE_ETH_EVENT_INTR_LSC,
						failsafe_eth_lsc_event_callback,
						sdev);
		if (ret)
			WARN("Failed to unregister LSC callback for sub_device"
			     " %d", SUB_ID(sdev));
		sdev->lsc_callback = 0;
	}
}

void
failsafe_dev_remove(struct rte_eth_dev *dev)
{
	struct sub_device *sdev;
	uint8_t i;

	FOREACH_SUBDEV(sdev, i, dev) {
		if (!sdev->remove)
			continue;

		/* Active devices must have finished their burst and
		 * their stats must be saved.
		 */
		if (sdev->state >= DEV_ACTIVE &&
		    fs_rxtx_clean(sdev) == 0)
			continue;
		if (fs_lock(dev, 1) != 0)
			return;
		if (sdev->state >= DEV_ACTIVE)
			fs_dev_stats_save(sdev);
		fs_dev_remove(sdev);
		fs_unlock(dev, 1);
	}
}

static int
failsafe_eth_dev_rx_queues_sync(struct rte_eth_dev *dev)
{
	struct rxq *rxq;
	int ret;
	uint16_t i;

	for (i = 0; i < dev->data->nb_rx_queues; i++) {
		rxq = dev->data->rx_queues[i];

		if (rxq->info.conf.rx_deferred_start &&
		    dev->data->rx_queue_state[i] ==
						RTE_ETH_QUEUE_STATE_STARTED) {
			/*
			 * The subdevice Rx queue does not launch on device
			 * start if deferred start flag is set. It needs to be
			 * started manually in case an appropriate failsafe Rx
			 * queue has been started earlier.
			 */
			ret = dev->dev_ops->rx_queue_start(dev, i);
			if (ret) {
				ERROR("Could not synchronize Rx queue %d", i);
				return ret;
			}
		} else if (dev->data->rx_queue_state[i] ==
						RTE_ETH_QUEUE_STATE_STOPPED) {
			/*
			 * The subdevice Rx queue needs to be stopped manually
			 * in case an appropriate failsafe Rx queue has been
			 * stopped earlier.
			 */
			ret = dev->dev_ops->rx_queue_stop(dev, i);
			if (ret) {
				ERROR("Could not synchronize Rx queue %d", i);
				return ret;
			}
		}
	}
	return 0;
}

static int
failsafe_eth_dev_tx_queues_sync(struct rte_eth_dev *dev)
{
	struct txq *txq;
	int ret;
	uint16_t i;

	for (i = 0; i < dev->data->nb_tx_queues; i++) {
		txq = dev->data->tx_queues[i];

		if (txq->info.conf.tx_deferred_start &&
		    dev->data->tx_queue_state[i] ==
						RTE_ETH_QUEUE_STATE_STARTED) {
			/*
			 * The subdevice Tx queue does not launch on device
			 * start if deferred start flag is set. It needs to be
			 * started manually in case an appropriate failsafe Tx
			 * queue has been started earlier.
			 */
			ret = dev->dev_ops->tx_queue_start(dev, i);
			if (ret) {
				ERROR("Could not synchronize Tx queue %d", i);
				return ret;
			}
		} else if (dev->data->tx_queue_state[i] ==
						RTE_ETH_QUEUE_STATE_STOPPED) {
			/*
			 * The subdevice Tx queue needs to be stopped manually
			 * in case an appropriate failsafe Tx queue has been
			 * stopped earlier.
			 */
			ret = dev->dev_ops->tx_queue_stop(dev, i);
			if (ret) {
				ERROR("Could not synchronize Tx queue %d", i);
				return ret;
			}
		}
	}
	return 0;
}

int
failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
{
	struct sub_device *sdev;
	uint32_t inactive;
	int ret;
	uint8_t i;

	if (PRIV(dev)->state < DEV_PARSED)
		return 0;

	ret = failsafe_args_parse_subs(dev);
	if (ret)
		goto err_remove;

	if (PRIV(dev)->state < DEV_PROBED)
		return 0;
	ret = failsafe_eal_init(dev);
	if (ret)
		goto err_remove;
	if (PRIV(dev)->state < DEV_ACTIVE)
		return 0;
	inactive = 0;
	FOREACH_SUBDEV(sdev, i, dev) {
		if (sdev->state == DEV_PROBED) {
			inactive |= UINT32_C(1) << i;
			ret = eth_dev_flow_isolate_set(dev, sdev);
			if (ret) {
				ERROR("Could not apply configuration to sub_device %d",
				      i);
				goto err_remove;
			}
		}
	}
	ret = dev->dev_ops->dev_configure(dev);
	if (ret)
		goto err_remove;
	FOREACH_SUBDEV(sdev, i, dev) {
		if (inactive & (UINT32_C(1) << i)) {
			ret = fs_eth_dev_conf_apply(dev, sdev);
			if (ret) {
				ERROR("Could not apply configuration to sub_device %d",
				      i);
				goto err_remove;
			}
		}
	}
	/*
	 * If new devices have been configured, check if
	 * the link state has changed.
	 */
	if (inactive)
		dev->dev_ops->link_update(dev, 1);
	if (PRIV(dev)->state < DEV_STARTED)
		return 0;
	ret = dev->dev_ops->dev_start(dev);
	if (ret)
		goto err_remove;
	ret = failsafe_eth_dev_rx_queues_sync(dev);
	if (ret)
		goto err_remove;
	ret = failsafe_eth_dev_tx_queues_sync(dev);
	if (ret)
		goto err_remove;
	return 0;
err_remove:
	FOREACH_SUBDEV(sdev, i, dev)
		if (sdev->state != PRIV(dev)->state)
			sdev->remove = 1;
	return ret;
}

void
failsafe_stats_increment(struct rte_eth_stats *to, struct rte_eth_stats *from)
{
	uint32_t i;

	RTE_ASSERT(to != NULL && from != NULL);
	to->ipackets += from->ipackets;
	to->opackets += from->opackets;
	to->ibytes += from->ibytes;
	to->obytes += from->obytes;
	to->imissed += from->imissed;
	to->ierrors += from->ierrors;
	to->oerrors += from->oerrors;
	to->rx_nombuf += from->rx_nombuf;
	for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
		to->q_ipackets[i] += from->q_ipackets[i];
		to->q_opackets[i] += from->q_opackets[i];
		to->q_ibytes[i] += from->q_ibytes[i];
		to->q_obytes[i] += from->q_obytes[i];
		to->q_errors[i] += from->q_errors[i];
	}
}

int
failsafe_eth_rmv_event_callback(uint16_t port_id __rte_unused,
				enum rte_eth_event_type event __rte_unused,
				void *cb_arg, void *out __rte_unused)
{
	struct sub_device *sdev = cb_arg;

	fs_lock(fs_dev(sdev), 0);
	/* Switch as soon as possible tx_dev. */
	fs_switch_dev(fs_dev(sdev), sdev);
	/* Use safe bursts in any case. */
	failsafe_set_burst_fn(fs_dev(sdev), 1);
	/*
	 * Async removal, the sub-PMD will try to unregister
	 * the callback at the source of the current thread context.
	 */
	sdev->remove = 1;
	fs_unlock(fs_dev(sdev), 0);
	return 0;
}

int
failsafe_eth_lsc_event_callback(uint16_t port_id __rte_unused,
				enum rte_eth_event_type event __rte_unused,
				void *cb_arg, void *out __rte_unused)
{
	struct rte_eth_dev *dev = cb_arg;
	int ret;

	ret = dev->dev_ops->link_update(dev, 0);
	/* We must pass on the LSC event */
	if (ret)
		return rte_eth_dev_callback_process(dev,
						    RTE_ETH_EVENT_INTR_LSC,
						    NULL);
	else
		return 0;
}

/* Take sub-device ownership before it becomes exposed to the application. */
int
failsafe_eth_new_event_callback(uint16_t port_id,
				enum rte_eth_event_type event __rte_unused,
				void *cb_arg, void *out __rte_unused)
{
	struct rte_eth_dev *fs_dev = cb_arg;
	struct sub_device *sdev;
	struct rte_eth_dev *dev = &rte_eth_devices[port_id];
	uint8_t i;

	FOREACH_SUBDEV_STATE(sdev, i, fs_dev, DEV_PARSED) {
		if (sdev->state >= DEV_PROBED)
			continue;
		if (dev->device == NULL) {
			WARN("Trying to probe malformed device %s.\n",
			     sdev->devargs.name);
			continue;
		}
		if (strcmp(sdev->devargs.name, dev->device->name) != 0)
			continue;
		rte_eth_dev_owner_set(port_id, &PRIV(fs_dev)->my_owner);
		/* The actual owner will be checked after the port probing. */
		break;
	}
	return 0;
}