thread: add spdk_thread_get_count()

This will return the number of currently allocated threads.

Modify the bdev_io caching code to use this new API since these
caches are really per-thread, not per-core.  SPDK does not support
dynamic threading yet, but once it does, we will want callers to be
using functions from the thread API - not counting the number of cores
allocated to the application.

spdk_env_get_core_count may still be useful as a helper function, so
it is still kept and not deprecated.  For example, app.c uses it to
print the number of cores allocated to the application.  bdevperf should
eventually be modified to use spdk_thread_get_count, but holding off on
that for now until spdk_event_allocate() uses threads instead of a reactor
lcore to specify where the event should be executed.

Signed-off-by: Jim Harris <james.r.harris@intel.com>
Change-Id: I5a30e3e825e6821da87d3927a2443768dfd740f4

Reviewed-on: https://review.gerrithub.io/414709
Tested-by: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: Madhu Pai <mpai@netapp.com>
Reviewed-by: Daniel Verkamp <daniel.verkamp@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

include/spdk/thread.h

@@ -191,6 +191,11 @@ struct spdk_thread *spdk_allocate_thread(spdk_thread_pass_msg msg_fn,
  */
 void spdk_free_thread(void);
 
+/**
+ * Get count of allocated threads.
+ */
+uint32_t spdk_thread_get_count(void);
+
 /**
  * Get a handle to the current thread.
  *

lib/bdev/bdev.c

@@ -635,10 +635,10 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg)
 
 	/**
 	 * Ensure no more than half of the total buffers end up local caches, by
-	 *   using spdk_env_get_core_count() to determine how many local caches we need
+	 *   using spdk_thread_get_count() to determine how many local caches we need
 	 *   to account for.
 	 */
-	cache_size = BUF_SMALL_POOL_SIZE / (2 * spdk_env_get_core_count());
+	cache_size = BUF_SMALL_POOL_SIZE / (2 * spdk_thread_get_count());
 	snprintf(mempool_name, sizeof(mempool_name), "buf_small_pool_%d", getpid());
 
 	g_bdev_mgr.buf_small_pool = spdk_mempool_create(mempool_name,
@@ -652,7 +652,7 @@ spdk_bdev_initialize(spdk_bdev_init_cb cb_fn, void *cb_arg)
 		return;
 	}
 
-	cache_size = BUF_LARGE_POOL_SIZE / (2 * spdk_env_get_core_count());
+	cache_size = BUF_LARGE_POOL_SIZE / (2 * spdk_thread_get_count());
 	snprintf(mempool_name, sizeof(mempool_name), "buf_large_pool_%d", getpid());
 
 	g_bdev_mgr.buf_large_pool = spdk_mempool_create(mempool_name,

lib/thread/thread.c

@@ -75,6 +75,7 @@ struct spdk_thread {
 };
 
 static TAILQ_HEAD(, spdk_thread) g_threads = TAILQ_HEAD_INITIALIZER(g_threads);
+static uint32_t g_thread_count = 0;
 
 static struct spdk_thread *
 _get_thread(void)
@@ -137,6 +138,7 @@ spdk_allocate_thread(spdk_thread_pass_msg msg_fn,
 	thread->thread_ctx = thread_ctx;
 	TAILQ_INIT(&thread->io_channels);
 	TAILQ_INSERT_TAIL(&g_threads, thread, tailq);
+	g_thread_count++;
 	if (name) {
 		_set_thread_name(name);
 		thread->name = strdup(name);
@@ -161,6 +163,8 @@ spdk_free_thread(void)
 		return;
 	}
 
+	assert(g_thread_count > 0);
+	g_thread_count--;
 	TAILQ_REMOVE(&g_threads, thread, tailq);
 	free(thread->name);
 	free(thread);
@@ -168,6 +172,17 @@ spdk_free_thread(void)
 	pthread_mutex_unlock(&g_devlist_mutex);
 }
 
+uint32_t
+spdk_thread_get_count(void)
+{
+	/*
+	 * Return cached value of the current thread count.  We could acquire the
+	 *  lock and iterate through the TAILQ of threads to count them, but that
+	 *  count could still be invalidated after we release the lock.
+	 */
+	return g_thread_count;
+}
+
 struct spdk_thread *
 spdk_get_thread(void)
 {

test/common/lib/test_env.c

@@ -316,9 +316,3 @@ spdk_pci_addr_compare(const struct spdk_pci_addr *a1, const struct spdk_pci_addr
 
 	return 0;
 }
-
-uint32_t
-spdk_env_get_core_count(void)
-{
-	return 1;
-}