From bd15f237f27c39bb94fa7dda5c1806c08520afe9 Mon Sep 17 00:00:00 2001
From: Long Li <longli@microsoft.com>
Date: Wed, 5 Oct 2022 16:21:55 -0700
Subject: [PATCH] net/mana: support device removal interrupts

MANA supports PCI hot plug events. Add this interrupt to DPDK core so its
parent PMD can detect device removal during Azure servicing or live
migration.

Signed-off-by: Long Li <longli@microsoft.com>
---
 doc/guides/nics/features/mana.ini |   1 +
 drivers/net/mana/mana.c           | 103 ++++++++++++++++++++++++++++++
 drivers/net/mana/mana.h           |   1 +
 3 files changed, 105 insertions(+)

diff --git a/doc/guides/nics/features/mana.ini b/doc/guides/nics/features/mana.ini
index 07ac2e2d4d..953640e969 100644
--- a/doc/guides/nics/features/mana.ini
+++ b/doc/guides/nics/features/mana.ini
@@ -5,6 +5,7 @@
 ;
 [Features]
 Link status          = P
+Removal event        = Y
 Multiprocess aware   = Y
 Linux                = Y
 x86-64               = Y
diff --git a/drivers/net/mana/mana.c b/drivers/net/mana/mana.c
index 2b1f6fcf1e..0b94776594 100644
--- a/drivers/net/mana/mana.c
+++ b/drivers/net/mana/mana.c
@@ -103,12 +103,18 @@ mana_dev_configure(struct rte_eth_dev *dev)
 	return 0;
 }
 
+static int mana_intr_uninstall(struct mana_priv *priv);
+
 static int
 mana_dev_close(struct rte_eth_dev *dev)
 {
 	struct mana_priv *priv = dev->data->dev_private;
 	int ret;
 
+	ret = mana_intr_uninstall(priv);
+	if (ret)
+		return ret;
+
 	ret = ibv_close_device(priv->ib_ctx);
 	if (ret) {
 		ret = errno;
@@ -341,6 +347,96 @@ mana_ibv_device_to_pci_addr(const struct ibv_device *device,
 	return 0;
 }
 
+/*
+ * Interrupt handler from IB layer to notify this device is being removed.
+ */
+static void
+mana_intr_handler(void *arg)
+{
+	struct mana_priv *priv = arg;
+	struct ibv_context *ctx = priv->ib_ctx;
+	struct ibv_async_event event;
+
+	/* Read and ack all messages from IB device */
+	while (true) {
+		if (ibv_get_async_event(ctx, &event))
+			break;
+
+		if (event.event_type == IBV_EVENT_DEVICE_FATAL) {
+			struct rte_eth_dev *dev;
+
+			dev = &rte_eth_devices[priv->port_id];
+			if (dev->data->dev_conf.intr_conf.rmv)
+				rte_eth_dev_callback_process(dev,
+					RTE_ETH_EVENT_INTR_RMV, NULL);
+		}
+
+		ibv_ack_async_event(&event);
+	}
+}
+
+static int
+mana_intr_uninstall(struct mana_priv *priv)
+{
+	int ret;
+
+	ret = rte_intr_callback_unregister(priv->intr_handle,
+					   mana_intr_handler, priv);
+	if (ret <= 0) {
+		DRV_LOG(ERR, "Failed to unregister intr callback ret %d", ret);
+		return ret;
+	}
+
+	rte_intr_instance_free(priv->intr_handle);
+
+	return 0;
+}
+
+static int
+mana_intr_install(struct mana_priv *priv)
+{
+	int ret, flags;
+	struct ibv_context *ctx = priv->ib_ctx;
+
+	priv->intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_SHARED);
+	if (!priv->intr_handle) {
+		DRV_LOG(ERR, "Failed to allocate intr_handle");
+		rte_errno = ENOMEM;
+		return -ENOMEM;
+	}
+
+	rte_intr_fd_set(priv->intr_handle, -1);
+
+	flags = fcntl(ctx->async_fd, F_GETFL);
+	ret = fcntl(ctx->async_fd, F_SETFL, flags | O_NONBLOCK);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to change async_fd to NONBLOCK");
+		goto free_intr;
+	}
+
+	rte_intr_fd_set(priv->intr_handle, ctx->async_fd);
+	rte_intr_type_set(priv->intr_handle, RTE_INTR_HANDLE_EXT);
+
+	ret = rte_intr_callback_register(priv->intr_handle,
+					 mana_intr_handler, priv);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to register intr callback");
+		rte_intr_fd_set(priv->intr_handle, -1);
+		goto restore_fd;
+	}
+
+	return 0;
+
+restore_fd:
+	fcntl(ctx->async_fd, F_SETFL, flags);
+
+free_intr:
+	rte_intr_instance_free(priv->intr_handle);
+	priv->intr_handle = NULL;
+
+	return ret;
+}
+
 static int
 mana_proc_priv_init(struct rte_eth_dev *dev)
 {
@@ -623,6 +719,13 @@ mana_probe_port(struct ibv_device *ibdev, struct ibv_device_attr_ex *dev_attr,
 
 	rte_eth_copy_pci_info(eth_dev, pci_dev);
 
+	/* Create async interrupt handler */
+	ret = mana_intr_install(priv);
+	if (ret) {
+		DRV_LOG(ERR, "Failed to install intr handler");
+		goto failed;
+	}
+
 	rte_spinlock_lock(&mana_shared_data->lock);
 	mana_shared_data->primary_cnt++;
 	rte_spinlock_unlock(&mana_shared_data->lock);
diff --git a/drivers/net/mana/mana.h b/drivers/net/mana/mana.h
index d5f9b2661d..f249ef1f66 100644
--- a/drivers/net/mana/mana.h
+++ b/drivers/net/mana/mana.h
@@ -35,6 +35,7 @@ struct mana_priv {
 	struct ibv_pd *ib_pd;
 	struct ibv_pd *ib_parent_pd;
 	void *db_page;
+	struct rte_intr_handle *intr_handle;
 	int max_rx_queues;
 	int max_tx_queues;
 	int max_rx_desc;