lib/thread: Fail spdk_thread_exit() if thread is already exiting

This is a preparation to support voluntary thread termination by
calling spdk_thread_exit().

Change spdk_thread_exit() to return -EINVAL if the thread is already
marked as exited.  This will be helpful to detect wrong call sequence
of voluntary thread termination.

Besides, update reactor shutdown and unit test framework shutdown
to incorporate this change accordingly.

Signed-off-by: Shuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
Change-Id: I2296c61e273bf4d9580656dcbc2da0e8a8f3bcf7
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/671
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Reviewed-by: Aleksey Marchuk <alexeymar@mellanox.com>

include/spdk/thread.h

@@ -224,13 +224,14 @@ void spdk_set_thread(struct spdk_thread *thread);
  * Mark the thread as exited, failing all future spdk_thread_poll() calls. May
  * only be called within an spdk poller or message.
  *
+ * All I/O channel references associated with the thread must be released using
+ * spdk_put_io_channel() prior to calling this function.
  *
  * \param thread The thread to destroy.
  *
- * All I/O channel references associated with the thread must be released using
- * spdk_put_io_channel() prior to calling this function.
+ * \return 0 on success, negated errno on failure.
  */
-void spdk_thread_exit(struct spdk_thread *thread);
+int spdk_thread_exit(struct spdk_thread *thread);
 
 /**
  * Returns whether the thread is marked as exited.

lib/event/reactor.c

@@ -312,6 +312,7 @@ _spdk_reactor_run(void *arg)
 	struct spdk_lw_thread	*lw_thread, *tmp;
 	char			thread_name[32];
 	uint64_t		rusage_period = 0;
+	int			rc __attribute__((unused));
 
 	SPDK_NOTICELOG("Reactor started on core %u\n", reactor->lcore);
 
@@ -353,7 +354,10 @@ _spdk_reactor_run(void *arg)
 		thread = spdk_thread_get_from_ctx(lw_thread);
 		TAILQ_REMOVE(&reactor->threads, lw_thread, link);
 		spdk_set_thread(thread);
-		spdk_thread_exit(thread);
+		if (!spdk_thread_is_exited(thread)) {
+			rc = spdk_thread_exit(thread);
+			assert(rc == 0);
+		}
 		spdk_thread_destroy(thread);
 	}
 

lib/thread/thread.c

@@ -345,14 +345,21 @@ spdk_set_thread(struct spdk_thread *thread)
 	tls_thread = thread;
 }
 
-void
+int
 spdk_thread_exit(struct spdk_thread *thread)
 {
 	SPDK_DEBUGLOG(SPDK_LOG_THREAD, "Exit thread %s\n", thread->name);
 
 	assert(tls_thread == thread);
 
+	if (thread->exit) {
+		SPDK_ERRLOG("thread %s is already marked as exited\n",
+			    thread->name);
+		return -EINVAL;
+	}
+
 	thread->exit = true;
+	return 0;
 }
 
 bool

test/common/lib/ut_multithread.c

@@ -102,10 +102,14 @@ void
 free_threads(void)
 {
 	uint32_t i;
+	int rc __attribute__((unused));
 
 	for (i = 0; i < g_ut_num_threads; i++) {
 		set_thread(i);
-		spdk_thread_exit(g_ut_threads[i].thread);
+		if (!spdk_thread_is_exited(g_ut_threads[i].thread)) {
+			rc = spdk_thread_exit(g_ut_threads[i].thread);
+			assert(rc == 0);
+		}
 		spdk_thread_destroy(g_ut_threads[i].thread);
 		g_ut_threads[i].thread = NULL;
 	}

test/unit/lib/thread/thread.c/thread_ut.c

@@ -754,7 +754,7 @@ thread_exit(void)
 	bool done1 = false, done2 = false;
 	int rc __attribute__((unused));
 
-	allocate_threads(3);
+	allocate_threads(4);
 
 	/* Test all pending messages are reaped for the thread marked as exited. */
 	set_thread(0);
@@ -811,6 +811,14 @@ thread_exit(void)
 	spdk_io_device_unregister(&g_device1, NULL);
 	poll_threads();
 
+	/* Test call spdk_thread_exit() is only once for a single thread. */
+	set_thread(3);
+
+	thread = spdk_get_thread();
+
+	CU_ASSERT(spdk_thread_exit(thread) == 0);
+	CU_ASSERT(spdk_thread_exit(thread) == -EINVAL);
+
 	free_threads();
 }