net/mlx5: split multi-thread flow handling per OS

multi-threaded flows feature uses pthread function pthread_key_create
but for Windows the destruction option in the function is unimplemented.

To resolve it, Windows will implement destruction mechanism to cleanup
mlx5_flow_workspace object for each terminated thread.

Linux flow will keep the current behavior.

Signed-off-by: Tal Shnaiderman <talshn@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
Acked-by: Khoa To <khot@microsoft.com>

drivers/net/mlx5/linux/meson.build

@@ -9,5 +9,6 @@ sources += files(
 	'mlx5_verbs.c',
 	'mlx5_mp_os.c',
 	'mlx5_vlan_os.c',
+	'mlx5_flow_os.c',
 )
 

drivers/net/mlx5/linux/mlx5_flow_os.c

@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#include "mlx5_flow_os.h"
+
+#include <rte_thread.h>
+
+/* Key of thread specific flow workspace data. */
+static rte_tls_key key_workspace;
+
+int
+mlx5_flow_os_init_workspace_once(void)
+{
+	if (rte_thread_tls_key_create(&key_workspace, flow_release_workspace)) {
+		DRV_LOG(ERR, "Can't create flow workspace data thread key.");
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+void *
+mlx5_flow_os_get_specific_workspace(void)
+{
+	return rte_thread_tls_value_get(key_workspace);
+}
+
+int
+mlx5_flow_os_set_specific_workspace(struct mlx5_flow_workspace *data)
+{
+	return rte_thread_tls_value_set(key_workspace, data);
+}
+
+void
+mlx5_flow_os_release_workspace(void)
+{
+	rte_thread_tls_key_delete(key_workspace);
+}

drivers/net/mlx5/mlx5.c

@@ -1004,6 +1004,11 @@ mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn,
 		err = rte_errno;
 		goto error;
 	}
+	if (LIST_EMPTY(&mlx5_dev_ctx_list)) {
+		err = mlx5_flow_os_init_workspace_once();
+		if (err)
+			goto error;
+	}
 	mlx5_flow_aging_init(sh);
 	mlx5_flow_counters_mng_init(sh);
 	mlx5_flow_ipool_create(sh, config);
@@ -1079,6 +1084,9 @@ mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh)
 	mlx5_mr_release_cache(&sh->share_cache);
 	/* Remove context from the global device list. */
 	LIST_REMOVE(sh, next);
+	/* Release flow workspaces objects on the last device. */
+	if (LIST_EMPTY(&mlx5_dev_ctx_list))
+		mlx5_flow_os_release_workspace();
 	pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex);
 	/*
 	 *  Ensure there is no async event handler installed.

drivers/net/mlx5/mlx5_flow.c

@@ -696,11 +696,6 @@ static struct mlx5_flow_tunnel_info tunnels_info[] = {
 	},
 };
 
-/* Key of thread specific flow workspace data. */
-static pthread_key_t key_workspace;
-
-/* Thread specific flow workspace data once initialization data. */
-static pthread_once_t key_workspace_init;
 
 
 /**
@@ -5722,7 +5717,7 @@ mlx5_flow_start_default(struct rte_eth_dev *dev)
 /**
  * Release key of thread specific flow workspace data.
  */
-static void
+void
 flow_release_workspace(void *data)
 {
 	struct mlx5_flow_workspace *wks = data;
@@ -5736,16 +5731,6 @@ flow_release_workspace(void *data)
 	}
 }
 
-/**
- * Initialize key of thread specific flow workspace data.
- */
-static void
-flow_alloc_workspace(void)
-{
-	if (pthread_key_create(&key_workspace, flow_release_workspace))
-		DRV_LOG(ERR, "Can't create flow workspace data thread key.");
-}
-
 /**
  * Get thread specific current flow workspace.
  *
@@ -5756,7 +5741,7 @@ mlx5_flow_get_thread_workspace(void)
 {
 	struct mlx5_flow_workspace *data;
 
-	data = pthread_getspecific(key_workspace);
+	data = mlx5_flow_os_get_specific_workspace();
 	MLX5_ASSERT(data && data->inuse);
 	if (!data || !data->inuse)
 		DRV_LOG(ERR, "flow workspace not initialized.");
@@ -5804,11 +5789,7 @@ mlx5_flow_push_thread_workspace(void)
 	struct mlx5_flow_workspace *curr;
 	struct mlx5_flow_workspace *data;
 
-	if (pthread_once(&key_workspace_init, flow_alloc_workspace)) {
-		DRV_LOG(ERR, "Failed to init flow workspace data thread key.");
-		return NULL;
-	}
-	curr = pthread_getspecific(key_workspace);
+	curr = mlx5_flow_os_get_specific_workspace();
 	if (!curr) {
 		data = flow_alloc_thread_workspace();
 		if (!data)
@@ -5827,7 +5808,7 @@ mlx5_flow_push_thread_workspace(void)
 	data->inuse = 1;
 	data->flow_idx = 0;
 	/* Set as current workspace */
-	if (pthread_setspecific(key_workspace, data))
+	if (mlx5_flow_os_set_specific_workspace(data))
 		DRV_LOG(ERR, "Failed to set flow workspace to thread.");
 	return data;
 }
@@ -5853,7 +5834,7 @@ mlx5_flow_pop_thread_workspace(void)
 	data->inuse = 0;
 	if (!data->prev)
 		return;
-	if (pthread_setspecific(key_workspace, data->prev))
+	if (mlx5_flow_os_set_specific_workspace(data->prev))
 		DRV_LOG(ERR, "Failed to set flow workspace to thread.");
 }
 

drivers/net/mlx5/mlx5_flow.h

@@ -1503,4 +1503,11 @@ void flow_dv_dest_array_remove_cb(struct mlx5_cache_list *list,
 				  struct mlx5_cache_entry *entry);
 struct mlx5_aso_age_action *flow_aso_age_get_by_idx(struct rte_eth_dev *dev,
 						    uint32_t age_idx);
+
+void flow_release_workspace(void *data);
+int mlx5_flow_os_init_workspace_once(void);
+void *mlx5_flow_os_get_specific_workspace(void);
+int mlx5_flow_os_set_specific_workspace(struct mlx5_flow_workspace *data);
+void mlx5_flow_os_release_workspace(void);
+
 #endif /* RTE_PMD_MLX5_FLOW_H_ */

drivers/net/mlx5/windows/mlx5_flow_os.c

@@ -5,6 +5,8 @@
 #include "mlx5_flow_os.h"
 #include "mlx5_win_ext.h"
 
+#include <rte_thread.h>
+
 /**
  * Verify the @p attributes will be correctly understood by the NIC and store
  * them in the @p flow if everything is correct.
@@ -238,3 +240,179 @@ mlx5_flow_os_destroy_flow(void *drv_flow_ptr)
 {
 	return mlx5_glue->devx_fs_rule_del(drv_flow_ptr);
 }
+
+struct mlx5_workspace_thread {
+	HANDLE	thread_handle;
+	struct mlx5_flow_workspace *mlx5_ws;
+	struct mlx5_workspace_thread *next;
+};
+
+/**
+ * Static pointer array for multi thread support of mlx5_flow_workspace.
+ */
+static struct mlx5_workspace_thread *curr;
+static struct mlx5_workspace_thread *first;
+rte_tls_key ws_tls_index;
+static pthread_mutex_t lock_thread_list;
+
+static bool
+mlx5_is_thread_alive(HANDLE thread_handle)
+{
+	DWORD result = WaitForSingleObject(thread_handle, 0);
+
+	if (result == WAIT_OBJECT_0)
+		return false;
+	return false;
+}
+
+static int
+mlx5_get_current_thread(HANDLE *p_handle)
+{
+	BOOL ret = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
+		GetCurrentProcess(), p_handle, 0, 0, DUPLICATE_SAME_ACCESS);
+
+	if (!ret) {
+		RTE_LOG_WIN32_ERR("DuplicateHandle()");
+		return -1;
+	}
+	return 0;
+}
+
+static void
+mlx5_clear_thread_list(void)
+{
+	struct mlx5_workspace_thread *temp = first;
+	struct mlx5_workspace_thread *next, *prev = NULL;
+	HANDLE curr_thread;
+
+	if (!temp)
+		return;
+	if (mlx5_get_current_thread(&curr_thread)) {
+		DRV_LOG(ERR, "Failed to get current thread "
+			"handle.");
+		return;
+	}
+	while (temp) {
+		next = temp->next;
+		if (temp->thread_handle != curr_thread &&
+		    !mlx5_is_thread_alive(temp->thread_handle)) {
+			if (temp == first) {
+				if (curr == temp)
+					curr = temp->next;
+				first = temp->next;
+			} else if (temp == curr) {
+				curr = prev;
+			}
+			flow_release_workspace(temp->mlx5_ws);
+			CloseHandle(temp->thread_handle);
+			free(temp);
+			if (prev)
+				prev->next = next;
+			temp = next;
+			continue;
+		}
+		prev = temp;
+		temp = temp->next;
+	}
+	CloseHandle(curr_thread);
+}
+
+/**
+ * Release workspaces before exit.
+ */
+void
+mlx5_flow_os_release_workspace(void)
+{
+	mlx5_clear_thread_list();
+	if (first) {
+		MLX5_ASSERT(!first->next);
+		flow_release_workspace(first->mlx5_ws);
+		free(first);
+	}
+	rte_thread_tls_key_delete(ws_tls_index);
+	pthread_mutex_destroy(&lock_thread_list);
+}
+
+static int
+mlx5_add_workspace_to_list(struct mlx5_flow_workspace *data)
+{
+	HANDLE curr_thread;
+	struct mlx5_workspace_thread *temp = calloc(1, sizeof(*temp));
+
+	if (!temp) {
+		DRV_LOG(ERR, "Failed to allocate thread workspace "
+			"memory.");
+		return -1;
+	}
+	if (mlx5_get_current_thread(&curr_thread)) {
+		DRV_LOG(ERR, "Failed to get current thread "
+			"handle.");
+		free(temp);
+		return -1;
+	}
+	temp->mlx5_ws = data;
+	temp->thread_handle = curr_thread;
+	pthread_mutex_lock(&lock_thread_list);
+	mlx5_clear_thread_list();
+	if (!first) {
+		first = temp;
+		curr = temp;
+	} else {
+		curr->next = temp;
+		curr = curr->next;
+	}
+	pthread_mutex_unlock(&lock_thread_list);
+	return 0;
+}
+
+int
+mlx5_flow_os_init_workspace_once(void)
+{
+	int err = rte_thread_tls_key_create(&ws_tls_index, NULL);
+
+	if (err) {
+		DRV_LOG(ERR, "Can't create flow workspace data thread key.");
+		return err;
+	}
+	pthread_mutex_init(&lock_thread_list, NULL);
+	return 0;
+}
+
+void *
+mlx5_flow_os_get_specific_workspace(void)
+{
+	return rte_thread_tls_value_get(ws_tls_index);
+}
+
+int
+mlx5_flow_os_set_specific_workspace(struct mlx5_flow_workspace *data)
+{
+	int err = 0;
+	int old_err = rte_errno;
+
+	rte_errno = 0;
+	if (!rte_thread_tls_value_get(ws_tls_index)) {
+		if (rte_errno) {
+			DRV_LOG(ERR, "Failed checking specific workspace.");
+			rte_errno = old_err;
+			return -1;
+		}
+		/*
+		 * set_specific_workspace when current value is NULL
+		 * can happen only once per thread, mark this thread in
+		 * linked list to be able to release reasorces later on.
+		 */
+		err = mlx5_add_workspace_to_list(data);
+		if (err) {
+			DRV_LOG(ERR, "Failed adding workspace to list.");
+			rte_errno = old_err;
+			return -1;
+		}
+	}
+	if (rte_thread_tls_value_set(ws_tls_index, data)) {
+		DRV_LOG(ERR, "Failed setting specific workspace.");
+		err = -1;
+	}
+	rte_errno = old_err;
+	return err;
+}