lib/event: cleanup and document scheduler API

This patch cleans up the header file, structures and
parameters of scheduler API. While documenting the
functionality.

- made scheduler name const
- removed typedefs for schedueler callbacks
- balance() now accepts uint32_t for array size instead of an int
- removed unused _spdk_lw_thread_set_core()
- renamed _spdk_scheduler_period_set() to _spdk_scheduler_set_period()
- renamed _spdk_scheduler_period_get() to _spdk_scheduler_get_period()
- renamed _spdk_scheduler_list_add() to _spdk_scheduler_register()

This is preparation to making this API public.

Signed-off-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Change-Id: Ia7b6b6a5eafb052ac275db6c04113a8ad442383f
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/8842
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: Jim Harris <james.r.harris@intel.com>
Reviewed-by: Konrad Sztyber <konrad.sztyber@gmail.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>

include/spdk_internal/event.h

@@ -255,40 +255,35 @@ struct spdk_scheduler_core_info {
 };
 
 /**
- * Scheduler balance function type.
- * Accepts array of core_info which is of size 'count' and returns updated array.
+ * Thread scheduler.
+ * Functions from this structure are invoked from scheduling reactor.
  */
-typedef void (*spdk_scheduler_balance_fn)(struct spdk_scheduler_core_info *core_info, int count);
-
-/**
- * Scheduler init function type.
- * Called on scheduler module initialization.
- */
-typedef int (*spdk_scheduler_init_fn)(void);
-
-/**
- * Scheduler deinitialization function type.
- * Called on reactor fini.
- */
-typedef void (*spdk_scheduler_deinit_fn)(void);
-
-/** Thread scheduler */
 struct spdk_scheduler {
-	char                        *name;
-	spdk_scheduler_init_fn       init;
-	spdk_scheduler_deinit_fn     deinit;
-	spdk_scheduler_balance_fn    balance;
-	TAILQ_ENTRY(spdk_scheduler)  link;
-};
+	const char *name;
 
-/**
- * Add the given scheduler to the list of registered schedulers.
- * This function should be invoked by referencing the macro
- * SPDK_SCHEDULER_REGISTER in the scheduler c file.
- *
- * \param scheduler Scheduler to be added.
- */
-void _spdk_scheduler_list_add(struct spdk_scheduler *scheduler);
+	/**
+	 * This function is called to initialize a scheduler.
+	 *
+	 * \return 0 on success or non-zero on failure.
+	 */
+	int (*init)(void);
+
+	/**
+	 * This function is called to deinitialize a scheduler.
+	 */
+	void (*deinit)(void);
+
+	/**
+	 * Function to balance threads across cores by modifying
+	 * the value of their lcore field.
+	 *
+	 * \param core_info Structure describing cores and threads on them.
+	 * \param count Size of the core_info array.
+	 */
+	void (*balance)(struct spdk_scheduler_core_info *core_info, uint32_t count);
+
+	TAILQ_ENTRY(spdk_scheduler)	link;
+};
 
 /**
  * Change current scheduler. If another scheduler was used prior,
@@ -300,44 +295,48 @@ void _spdk_scheduler_list_add(struct spdk_scheduler *scheduler);
  *
  * \return 0 on success or non-zero on failure.
  */
-int _spdk_scheduler_set(char *name);
+int _spdk_scheduler_set(const char *name);
 
 /**
  * Get currently set scheduler.
  *
+ * \return a pointer to spdk scheduler or NULL if none is set.
  */
 struct spdk_scheduler *_spdk_scheduler_get(void);
 
 /**
  * Change current scheduling period.
+ * Setting period to 0 disables scheduling.
  *
- * \param period New period (microseconds).
+ * \param period Period to set in microseconds.
  */
-void _spdk_scheduler_period_set(uint64_t period);
+void _spdk_scheduler_set_period(uint64_t period);
 
 /**
- * Get period of currently set scheduler.
+ * Get scheduling period of currently set scheduler.
+ *
+ * \return Scheduling period in microseconds.
  */
-uint64_t _spdk_scheduler_period_get(void);
+uint64_t _spdk_scheduler_get_period(void);
+
+/**
+ * Add the given scheduler to the list of registered schedulers.
+ * This function should be invoked by referencing the macro
+ * SPDK_SCHEDULER_REGISTER in the scheduler c file.
+ *
+ * \param scheduler Scheduler to be added.
+ */
+void _spdk_scheduler_register(struct spdk_scheduler *scheduler);
 
 /*
- * Macro used to register new reactor balancer.
+ * Macro used to register new scheduler.
  */
 #define SPDK_SCHEDULER_REGISTER(scheduler) \
 static void __attribute__((constructor)) _spdk_scheduler_register_ ## scheduler (void) \
 { \
-	_spdk_scheduler_list_add(&scheduler); \
+	_spdk_scheduler_register(&scheduler); \
 } \
 
-/**
- * Set new CPU core index. Used for scheduling, assigns new CPU core index and marks it =
- * for rescheduling - does not actually change it. Can be used with SPDK_ENV_LCORE_ID_ANY
- *
- * \param thread thread to change core.
- * \param lcore new CPU core index.
- */
-void _spdk_lw_thread_set_core(struct spdk_lw_thread *thread, uint32_t lcore);
-
 #ifdef __cplusplus
 }
 #endif

lib/event/app.c

@@ -620,7 +620,7 @@ app_stop(void *arg1)
 
 	spdk_rpc_finish();
 	g_spdk_app.stopped = true;
-	_spdk_scheduler_period_set(0);
+	_spdk_scheduler_set_period(0);
 	_start_subsystem_fini(NULL);
 }
 

lib/event/app_rpc.c

@@ -491,7 +491,7 @@ rpc_framework_set_scheduler(struct spdk_jsonrpc_request *request,
 	}
 
 	if (req.period != 0) {
-		_spdk_scheduler_period_set(req.period);
+		_spdk_scheduler_set_period(req.period);
 	}
 
 	ret = _spdk_scheduler_set(req.name);
@@ -514,7 +514,7 @@ rpc_framework_get_scheduler(struct spdk_jsonrpc_request *request,
 {
 	struct spdk_json_write_ctx *w;
 	struct spdk_scheduler *scheduler = _spdk_scheduler_get();
-	uint64_t scheduler_period = _spdk_scheduler_period_get();
+	uint64_t scheduler_period = _spdk_scheduler_get_period();
 	struct spdk_governor *governor = _spdk_governor_get();
 
 	if (params) {

lib/event/gscheduler.c

@@ -53,7 +53,7 @@ deinit(void)
 }
 
 static void
-balance(struct spdk_scheduler_core_info *cores, int core_count)
+balance(struct spdk_scheduler_core_info *cores, uint32_t core_count)
 {
 	struct spdk_governor *governor;
 	struct spdk_scheduler_core_info *core;

lib/event/reactor.c

@@ -82,7 +82,7 @@ static int reactor_interrupt_init(struct spdk_reactor *reactor);
 static void reactor_interrupt_fini(struct spdk_reactor *reactor);
 
 static struct spdk_scheduler *
-_scheduler_find(char *name)
+_scheduler_find(const char *name)
 {
 	struct spdk_scheduler *tmp;
 
@@ -96,7 +96,7 @@ _scheduler_find(char *name)
 }
 
 int
-_spdk_scheduler_set(char *name)
+_spdk_scheduler_set(const char *name)
 {
 	struct spdk_scheduler *scheduler;
 	int rc = 0;
@@ -138,21 +138,21 @@ _spdk_scheduler_get(void)
 }
 
 uint64_t
-_spdk_scheduler_period_get(void)
+_spdk_scheduler_get_period(void)
 {
 	/* Convert from ticks to microseconds */
 	return (g_scheduler_period * SPDK_SEC_TO_USEC / spdk_get_ticks_hz());
 }
 
 void
-_spdk_scheduler_period_set(uint64_t period)
+_spdk_scheduler_set_period(uint64_t period)
 {
 	/* Convert microseconds to ticks */
 	g_scheduler_period = period * spdk_get_ticks_hz() / SPDK_SEC_TO_USEC;
 }
 
 void
-_spdk_scheduler_list_add(struct spdk_scheduler *scheduler)
+_spdk_scheduler_register(struct spdk_scheduler *scheduler)
 {
 	if (_scheduler_find(scheduler->name)) {
 		SPDK_ERRLOG("scheduler named '%s' already registered.\n", scheduler->name);

lib/event/scheduler_dynamic.c

@@ -273,7 +273,7 @@ _balance_active(struct spdk_scheduler_thread_info *thread_info)
 }
 
 static void
-balance(struct spdk_scheduler_core_info *cores_info, int cores_count)
+balance(struct spdk_scheduler_core_info *cores_info, uint32_t cores_count)
 {
 	struct spdk_reactor *reactor;
 	struct spdk_governor *governor;

lib/event/scheduler_static.c

@@ -51,7 +51,7 @@ deinit_static(void)
 }
 
 static void
-balance_static(struct spdk_scheduler_core_info *cores, int core_count)
+balance_static(struct spdk_scheduler_core_info *cores, uint32_t core_count)
 {
 }
 

test/unit/lib/event/app.c/app_ut.c

@@ -58,7 +58,7 @@ DEFINE_STUB_V(spdk_reactors_start, (void));
 DEFINE_STUB_V(spdk_reactors_stop, (void *arg1));
 DEFINE_STUB(spdk_reactors_init, int, (void), 0);
 DEFINE_STUB_V(spdk_reactors_fini, (void));
-DEFINE_STUB_V(_spdk_scheduler_period_set, (uint64_t period));
+DEFINE_STUB_V(_spdk_scheduler_set_period, (uint64_t period));
 bool g_scheduling_in_progress;
 
 static void