net/bnxt: support ULP session manager cleanup

A ULP session will contain all the resources needed to support
rte flow offloads. A session is initialized as part of rte_eth_device
start. A DPDK application can have multiple interfaces which
means rte_eth_device start will be called for each of these devices.
ULP session manager will make sure that a single ULP session is only
initialized once. Apart from this, it also initializes MARK database,
EEM table & flow database. ULP session manager also manages a list of
all opened ULP sessions.

This patch adds support for cleaning up resources initialized for ULP
sessions.

Signed-off-by: Venkat Duvvuru <venkatkumar.duvvuru@broadcom.com>
Signed-off-by: Mike Baucom <michael.baucom@broadcom.com>
Reviewed-by: Lance Richardson <lance.richardson@broadcom.com>
Reviewed-by: Ajit Khaparde <ajit.khaparde@broadcom.com>

drivers/net/bnxt/bnxt_ethdev.c

@@ -951,6 +951,9 @@ static void bnxt_dev_stop_op(struct rte_eth_dev *eth_dev)
 	struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 
+	if (bp->truflow)
+		bnxt_ulp_deinit(bp);
+
 	eth_dev->data->dev_started = 0;
 	/* Prevent crashes when queues are still in use */
 	eth_dev->rx_pkt_burst = &bnxt_dummy_recv_pkts;

drivers/net/bnxt/tf_ulp/bnxt_ulp.c

@@ -27,6 +27,27 @@ STAILQ_HEAD(, bnxt_ulp_session_state) bnxt_ulp_session_list =
 /* Mutex to synchronize bnxt_ulp_session_list operations. */
 static pthread_mutex_t bnxt_ulp_global_mutex = PTHREAD_MUTEX_INITIALIZER;
 
+/*
+ * Allow the deletion of context only for the bnxt device that
+ * created the session
+ * TBD - The implementation of the function should change to
+ * using the reference count once tf_session_attach functionality
+ * is fixed.
+ */
+bool
+ulp_ctx_deinit_allowed(void *ptr)
+{
+	struct bnxt *bp = (struct bnxt *)ptr;
+
+	if (!bp)
+		return 0;
+
+	if (&bp->tfp == bp->ulp_ctx.g_tfp)
+		return 1;
+
+	return 0;
+}
+
 /*
  * Initialize an ULP session.
  * An ULP session will contain all the resources needed to support rte flow
@@ -67,6 +88,22 @@ ulp_ctx_session_open(struct bnxt *bp,
 	return rc;
 }
 
+/*
+ * Close the ULP session.
+ * It takes the ulp context pointer.
+ */
+static void
+ulp_ctx_session_close(struct bnxt *bp,
+		      struct bnxt_ulp_session_state *session)
+{
+	/* close the session in the hardware */
+	if (session->session_opened)
+		tf_close_session(&bp->tfp);
+	session->session_opened = 0;
+	session->g_tfp = NULL;
+	bp->ulp_ctx.g_tfp = NULL;
+}
+
 static void
 bnxt_init_tbl_scope_parms(struct bnxt *bp,
 			  struct tf_alloc_tbl_scope_parms *params)
@@ -138,6 +175,41 @@ ulp_eem_tbl_scope_init(struct bnxt *bp)
 	return 0;
 }
 
+/* Free Extended Exact Match host memory */
+static int32_t
+ulp_eem_tbl_scope_deinit(struct bnxt *bp, struct bnxt_ulp_context *ulp_ctx)
+{
+	struct tf_free_tbl_scope_parms	params = {0};
+	struct tf			*tfp;
+	int32_t				rc = 0;
+
+	if (!ulp_ctx || !ulp_ctx->cfg_data)
+		return -EINVAL;
+
+	/* Free the resources for the last device */
+	if (!ulp_ctx_deinit_allowed(bp))
+		return rc;
+
+	tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx);
+	if (!tfp) {
+		BNXT_TF_DBG(ERR, "Failed to get the truflow pointer\n");
+		return -EINVAL;
+	}
+
+	rc = bnxt_ulp_cntxt_tbl_scope_id_get(ulp_ctx, &params.tbl_scope_id);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "Failed to get the table scope id\n");
+		return -EINVAL;
+	}
+
+	rc = tf_free_tbl_scope(tfp, &params);
+	if (rc) {
+		BNXT_TF_DBG(ERR, "Unable to free table scope\n");
+		return -EINVAL;
+	}
+	return rc;
+}
+
 /* The function to free and deinit the ulp context data. */
 static int32_t
 ulp_ctx_deinit(struct bnxt *bp,
@@ -148,6 +220,9 @@ ulp_ctx_deinit(struct bnxt *bp,
 		return -EINVAL;
 	}
 
+	/* close the tf session */
+	ulp_ctx_session_close(bp, session);
+
 	/* Free the contents */
 	if (session->cfg_data) {
 		rte_free(session->cfg_data);
@@ -211,6 +286,36 @@ ulp_ctx_attach(struct bnxt_ulp_context *ulp_ctx,
 	return 0;
 }
 
+static int32_t
+ulp_ctx_detach(struct bnxt *bp,
+	       struct bnxt_ulp_session_state *session)
+{
+	struct bnxt_ulp_context *ulp_ctx;
+
+	if (!bp || !session) {
+		BNXT_TF_DBG(ERR, "Invalid Arguments\n");
+		return -EINVAL;
+	}
+	ulp_ctx = &bp->ulp_ctx;
+
+	if (!ulp_ctx->cfg_data)
+		return 0;
+
+	/* TBD call TF_session_detach */
+
+	/* Increment the ulp context data reference count usage. */
+	if (ulp_ctx->cfg_data->ref_cnt >= 1) {
+		ulp_ctx->cfg_data->ref_cnt--;
+		if (ulp_ctx_deinit_allowed(bp))
+			ulp_ctx_deinit(bp, session);
+		ulp_ctx->cfg_data = NULL;
+		ulp_ctx->g_tfp = NULL;
+		return 0;
+	}
+	BNXT_TF_DBG(ERR, "context deatach on invalid data\n");
+	return 0;
+}
+
 /*
  * Initialize the state of an ULP session.
  * If the state of an ULP session is not initialized, set it's state to
@@ -296,6 +401,26 @@ ulp_session_init(struct bnxt *bp,
 	return session;
 }
 
+/*
+ * When a device is closed, remove it's associated session from the global
+ * session list.
+ */
+static void
+ulp_session_deinit(struct bnxt_ulp_session_state *session)
+{
+	if (!session)
+		return;
+
+	if (!session->cfg_data) {
+		pthread_mutex_lock(&bnxt_ulp_global_mutex);
+		STAILQ_REMOVE(&bnxt_ulp_session_list, session,
+			      bnxt_ulp_session_state, next);
+		pthread_mutex_destroy(&session->bnxt_ulp_mutex);
+		rte_free(session);
+		pthread_mutex_unlock(&bnxt_ulp_global_mutex);
+	}
+}
+
 /*
  * When a port is initialized by dpdk. This functions is called
  * and this function initializes the ULP context and rest of the
@@ -363,12 +488,52 @@ bnxt_ulp_init(struct bnxt *bp)
 	return rc;
 
 jump_to_error:
+	bnxt_ulp_deinit(bp);
 	return -ENOMEM;
 }
 
 /* Below are the access functions to access internal data of ulp context. */
 
-/* Function to set the Mark DB into the context. */
+/*
+ * When a port is deinit'ed by dpdk. This function is called
+ * and this function clears the ULP context and rest of the
+ * infrastructure associated with it.
+ */
+void
+bnxt_ulp_deinit(struct bnxt *bp)
+{
+	struct bnxt_ulp_session_state	*session;
+	struct rte_pci_device		*pci_dev;
+	struct rte_pci_addr		*pci_addr;
+
+	/* Get the session first */
+	pci_dev = RTE_DEV_TO_PCI(bp->eth_dev->device);
+	pci_addr = &pci_dev->addr;
+	pthread_mutex_lock(&bnxt_ulp_global_mutex);
+	session = ulp_get_session(pci_addr);
+	pthread_mutex_unlock(&bnxt_ulp_global_mutex);
+
+	/* session not found then just exit */
+	if (!session)
+		return;
+
+	/* cleanup the eem table scope */
+	ulp_eem_tbl_scope_deinit(bp, &bp->ulp_ctx);
+
+	/* cleanup the flow database */
+	ulp_flow_db_deinit(&bp->ulp_ctx);
+
+	/* Delete the Mark database */
+	ulp_mark_db_deinit(&bp->ulp_ctx);
+
+	/* Delete the ulp context and tf session */
+	ulp_ctx_detach(bp, session);
+
+	/* Finally delete the bnxt session*/
+	ulp_session_deinit(session);
+}
+
+/* Function to set the Mark DB into the context */
 int32_t
 bnxt_ulp_cntxt_ptr2_mark_db_set(struct bnxt_ulp_context *ulp_ctx,
 				struct bnxt_ulp_mark_tbl *mark_tbl)

drivers/net/bnxt/tf_ulp/bnxt_ulp.h

@@ -47,6 +47,16 @@ struct rte_tf_flow {
 	uint32_t	flow_id;
 };
 
+/*
+ * Allow the deletion of context only for the bnxt device that
+ * created the session
+ * TBD - The implementation of the function should change to
+ * using the reference count once tf_session_attach functionality
+ * is fixed.
+ */
+bool
+ulp_ctx_deinit_allowed(void *bp);
+
 /* Function to set the device id of the hardware. */
 int32_t
 bnxt_ulp_cntxt_dev_id_set(struct bnxt_ulp_context *ulp_ctx, uint32_t dev_id);

drivers/net/bnxt/tf_ulp/ulp_mark_mgr.c

@@ -92,3 +92,28 @@ ulp_mark_db_init(struct bnxt_ulp_context *ctxt)
 
 	return -ENOMEM;
 }
+
+/*
+ * Release all resources in the Mark Manager for this ulp context
+ *
+ * ctxt [in] The ulp context for the mark manager
+ *
+ */
+int32_t
+ulp_mark_db_deinit(struct bnxt_ulp_context *ctxt)
+{
+	struct bnxt_ulp_mark_tbl *mtbl;
+
+	mtbl = bnxt_ulp_cntxt_ptr2_mark_db_get(ctxt);
+
+	if (mtbl) {
+		rte_free(mtbl->gfid_tbl);
+		rte_free(mtbl->lfid_tbl);
+		rte_free(mtbl);
+
+		/* Safe to ignore on deinit */
+		(void)bnxt_ulp_cntxt_ptr2_mark_db_set(ctxt, NULL);
+	}
+
+	return 0;
+}

drivers/net/bnxt/tf_ulp/ulp_mark_mgr.h

@@ -46,4 +46,12 @@ struct bnxt_ulp_mark_tbl {
 int32_t
 ulp_mark_db_init(struct bnxt_ulp_context *ctxt);
 
+/*
+ * Release all resources in the Mark Manager for this ulp context
+ *
+ * ctxt [in] The ulp context for the mark manager
+ */
+int32_t
+ulp_mark_db_deinit(struct bnxt_ulp_context *ctxt);
+
 #endif /* _ULP_MARK_MGR_H_ */