2016-05-24 18:04:20 +00:00
|
|
|
/*-
|
|
|
|
* BSD LICENSE
|
|
|
|
*
|
|
|
|
* Copyright (c) Intel Corporation.
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
*
|
|
|
|
* * Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* * Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in
|
|
|
|
* the documentation and/or other materials provided with the
|
|
|
|
* distribution.
|
|
|
|
* * Neither the name of Intel Corporation nor the names of its
|
|
|
|
* contributors may be used to endorse or promote products derived
|
|
|
|
* from this software without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
|
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
2017-05-02 18:18:25 +00:00
|
|
|
#include "spdk/stdinc.h"
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2017-05-02 18:18:25 +00:00
|
|
|
#include "spdk_internal/event.h"
|
2017-08-17 01:59:06 +00:00
|
|
|
#include "spdk_internal/log.h"
|
2016-05-24 18:04:20 +00:00
|
|
|
|
|
|
|
#include "spdk/log.h"
|
2016-09-20 23:04:51 +00:00
|
|
|
#include "spdk/io_channel.h"
|
2016-08-18 19:52:48 +00:00
|
|
|
#include "spdk/env.h"
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2016-06-28 03:24:36 +00:00
|
|
|
#define SPDK_MAX_SOCKET 64
|
|
|
|
|
2017-06-02 18:40:40 +00:00
|
|
|
#define SPDK_MAX_REACTORS 128
|
2017-01-12 21:01:26 +00:00
|
|
|
#define SPDK_REACTOR_SPIN_TIME_US 1000
|
2017-01-20 17:57:27 +00:00
|
|
|
#define SPDK_TIMER_POLL_ITERATIONS 5
|
2016-11-02 20:31:33 +00:00
|
|
|
#define SPDK_EVENT_BATCH_SIZE 8
|
|
|
|
|
2016-09-30 21:00:16 +00:00
|
|
|
enum spdk_poller_state {
|
|
|
|
/* The poller is registered with a reactor but not currently executing its fn. */
|
|
|
|
SPDK_POLLER_STATE_WAITING,
|
|
|
|
|
|
|
|
/* The poller is currently running its fn. */
|
|
|
|
SPDK_POLLER_STATE_RUNNING,
|
|
|
|
|
|
|
|
/* The poller was unregistered during the execution of its fn. */
|
|
|
|
SPDK_POLLER_STATE_UNREGISTERED,
|
|
|
|
};
|
|
|
|
|
2016-08-11 23:00:45 +00:00
|
|
|
struct spdk_poller {
|
|
|
|
TAILQ_ENTRY(spdk_poller) tailq;
|
|
|
|
uint32_t lcore;
|
2016-09-30 21:00:16 +00:00
|
|
|
|
|
|
|
/* Current state of the poller; should only be accessed from the poller's thread. */
|
|
|
|
enum spdk_poller_state state;
|
|
|
|
|
2016-08-11 23:00:45 +00:00
|
|
|
uint64_t period_ticks;
|
|
|
|
uint64_t next_run_tick;
|
|
|
|
spdk_poller_fn fn;
|
|
|
|
void *arg;
|
2016-09-30 21:00:16 +00:00
|
|
|
|
|
|
|
struct spdk_event *unregister_complete_event;
|
2016-08-11 23:00:45 +00:00
|
|
|
};
|
|
|
|
|
2016-05-24 18:04:20 +00:00
|
|
|
enum spdk_reactor_state {
|
|
|
|
SPDK_REACTOR_STATE_INVALID = 0,
|
|
|
|
SPDK_REACTOR_STATE_INITIALIZED = 1,
|
|
|
|
SPDK_REACTOR_STATE_RUNNING = 2,
|
|
|
|
SPDK_REACTOR_STATE_EXITING = 3,
|
|
|
|
SPDK_REACTOR_STATE_SHUTDOWN = 4,
|
|
|
|
};
|
|
|
|
|
|
|
|
struct spdk_reactor {
|
|
|
|
/* Logical core number for this reactor. */
|
2016-07-26 19:56:07 +00:00
|
|
|
uint32_t lcore;
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2017-01-20 17:41:21 +00:00
|
|
|
/* Socket ID for this reactor. */
|
|
|
|
uint32_t socket_id;
|
2017-08-31 03:51:19 +00:00
|
|
|
|
2017-08-17 01:59:06 +00:00
|
|
|
/* Poller for get the rusage for the reactor. */
|
|
|
|
struct spdk_poller *rusage_poller;
|
2017-01-20 17:41:21 +00:00
|
|
|
|
2017-08-17 01:59:06 +00:00
|
|
|
/* The last known rusage values */
|
|
|
|
struct rusage rusage;
|
2017-08-31 03:51:19 +00:00
|
|
|
|
2016-05-24 18:04:20 +00:00
|
|
|
/*
|
|
|
|
* Contains pollers actively running on this reactor. Pollers
|
|
|
|
* are run round-robin. The reactor takes one poller from the head
|
|
|
|
* of the ring, executes it, then puts it back at the tail of
|
|
|
|
* the ring.
|
|
|
|
*/
|
2016-07-26 19:56:07 +00:00
|
|
|
TAILQ_HEAD(, spdk_poller) active_pollers;
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2016-07-26 19:56:07 +00:00
|
|
|
/**
|
|
|
|
* Contains pollers running on this reactor with a periodic timer.
|
|
|
|
*/
|
|
|
|
TAILQ_HEAD(timer_pollers_head, spdk_poller) timer_pollers;
|
|
|
|
|
2017-05-09 17:48:16 +00:00
|
|
|
struct spdk_ring *events;
|
2016-08-16 19:51:25 +00:00
|
|
|
|
2017-01-20 17:41:21 +00:00
|
|
|
/* Pointer to the per-socket g_spdk_event_mempool for this reactor. */
|
|
|
|
struct spdk_mempool *event_mempool;
|
|
|
|
|
2016-08-16 19:51:25 +00:00
|
|
|
uint64_t max_delay_us;
|
2017-01-05 21:23:20 +00:00
|
|
|
} __attribute__((aligned(64)));
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2017-06-02 18:40:40 +00:00
|
|
|
static struct spdk_reactor g_reactors[SPDK_MAX_REACTORS];
|
2016-05-24 18:04:20 +00:00
|
|
|
|
|
|
|
static enum spdk_reactor_state g_reactor_state = SPDK_REACTOR_STATE_INVALID;
|
|
|
|
|
2017-08-31 03:51:19 +00:00
|
|
|
static bool g_context_switch_monitor_enabled = true;
|
|
|
|
|
2016-08-16 19:51:25 +00:00
|
|
|
static void spdk_reactor_construct(struct spdk_reactor *w, uint32_t lcore,
|
|
|
|
uint64_t max_delay_us);
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2017-01-05 21:13:38 +00:00
|
|
|
static struct spdk_mempool *g_spdk_event_mempool[SPDK_MAX_SOCKET];
|
2016-05-24 18:04:20 +00:00
|
|
|
|
|
|
|
static struct spdk_reactor *
|
|
|
|
spdk_reactor_get(uint32_t lcore)
|
|
|
|
{
|
|
|
|
struct spdk_reactor *reactor;
|
|
|
|
reactor = &g_reactors[lcore];
|
|
|
|
return reactor;
|
|
|
|
}
|
|
|
|
|
2017-01-05 01:24:18 +00:00
|
|
|
struct spdk_event *
|
2017-01-05 01:40:14 +00:00
|
|
|
spdk_event_allocate(uint32_t lcore, spdk_event_fn fn, void *arg1, void *arg2)
|
2016-05-24 18:04:20 +00:00
|
|
|
{
|
|
|
|
struct spdk_event *event = NULL;
|
2017-01-20 18:09:43 +00:00
|
|
|
struct spdk_reactor *reactor = spdk_reactor_get(lcore);
|
2016-11-02 20:31:33 +00:00
|
|
|
|
2017-01-20 18:09:43 +00:00
|
|
|
event = spdk_mempool_get(reactor->event_mempool);
|
2016-09-30 21:14:18 +00:00
|
|
|
if (event == NULL) {
|
2016-08-18 16:45:50 +00:00
|
|
|
assert(false);
|
|
|
|
return NULL;
|
|
|
|
}
|
2016-05-24 18:04:20 +00:00
|
|
|
|
|
|
|
event->lcore = lcore;
|
|
|
|
event->fn = fn;
|
|
|
|
event->arg1 = arg1;
|
|
|
|
event->arg2 = arg2;
|
|
|
|
|
|
|
|
return event;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2017-01-05 01:24:18 +00:00
|
|
|
spdk_event_call(struct spdk_event *event)
|
2016-05-24 18:04:20 +00:00
|
|
|
{
|
|
|
|
int rc;
|
|
|
|
struct spdk_reactor *reactor;
|
|
|
|
|
|
|
|
reactor = spdk_reactor_get(event->lcore);
|
|
|
|
|
2016-08-18 16:45:50 +00:00
|
|
|
assert(reactor->events != NULL);
|
2017-05-09 17:48:16 +00:00
|
|
|
rc = spdk_ring_enqueue(reactor->events, (void **)&event, 1);
|
|
|
|
if (rc != 1) {
|
2016-08-18 16:45:50 +00:00
|
|
|
assert(false);
|
|
|
|
}
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
2017-01-20 17:41:21 +00:00
|
|
|
static inline uint32_t
|
|
|
|
_spdk_event_queue_run_batch(struct spdk_reactor *reactor)
|
2016-05-24 18:04:20 +00:00
|
|
|
{
|
2016-11-02 20:31:33 +00:00
|
|
|
unsigned count, i;
|
|
|
|
void *events[SPDK_EVENT_BATCH_SIZE];
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2017-01-20 17:41:21 +00:00
|
|
|
#ifdef DEBUG
|
|
|
|
/*
|
2017-05-09 17:48:16 +00:00
|
|
|
* spdk_ring_dequeue() fills events and returns how many entries it wrote,
|
2017-01-20 17:41:21 +00:00
|
|
|
* so we will never actually read uninitialized data from events, but just to be sure
|
|
|
|
* (and to silence a static analyzer false positive), initialize the array to NULL pointers.
|
|
|
|
*/
|
|
|
|
memset(events, 0, sizeof(events));
|
|
|
|
#endif
|
2017-05-09 17:48:16 +00:00
|
|
|
|
|
|
|
count = spdk_ring_dequeue(reactor->events, events, SPDK_EVENT_BATCH_SIZE);
|
2016-11-02 20:31:33 +00:00
|
|
|
if (count == 0) {
|
|
|
|
return 0;
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
2016-11-02 20:31:33 +00:00
|
|
|
for (i = 0; i < count; i++) {
|
|
|
|
struct spdk_event *event = events[i];
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2017-01-20 17:41:21 +00:00
|
|
|
assert(event != NULL);
|
2017-01-05 01:19:02 +00:00
|
|
|
event->fn(event->arg1, event->arg2);
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
2017-01-20 17:41:21 +00:00
|
|
|
spdk_mempool_put_bulk(reactor->event_mempool, events, count);
|
2016-08-16 19:51:25 +00:00
|
|
|
|
|
|
|
return count;
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
2016-07-26 19:56:07 +00:00
|
|
|
static void
|
|
|
|
spdk_poller_insert_timer(struct spdk_reactor *reactor, struct spdk_poller *poller, uint64_t now)
|
|
|
|
{
|
|
|
|
struct spdk_poller *iter;
|
|
|
|
uint64_t next_run_tick;
|
|
|
|
|
|
|
|
next_run_tick = now + poller->period_ticks;
|
|
|
|
poller->next_run_tick = next_run_tick;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Insert poller in the reactor's timer_pollers list in sorted order by next scheduled
|
|
|
|
* run time.
|
|
|
|
*/
|
|
|
|
TAILQ_FOREACH_REVERSE(iter, &reactor->timer_pollers, timer_pollers_head, tailq) {
|
|
|
|
if (iter->next_run_tick <= next_run_tick) {
|
|
|
|
TAILQ_INSERT_AFTER(&reactor->timer_pollers, iter, poller, tailq);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/* No earlier pollers were found, so this poller must be the new head */
|
|
|
|
TAILQ_INSERT_HEAD(&reactor->timer_pollers, poller, tailq);
|
|
|
|
}
|
|
|
|
|
2016-09-30 21:00:16 +00:00
|
|
|
static void
|
|
|
|
_spdk_poller_unregister_complete(struct spdk_poller *poller)
|
|
|
|
{
|
|
|
|
if (poller->unregister_complete_event) {
|
|
|
|
spdk_event_call(poller->unregister_complete_event);
|
|
|
|
}
|
|
|
|
|
|
|
|
free(poller);
|
|
|
|
}
|
|
|
|
|
2017-05-17 15:58:16 +00:00
|
|
|
static void
|
|
|
|
_spdk_reactor_msg_passed(void *arg1, void *arg2)
|
|
|
|
{
|
|
|
|
spdk_thread_fn fn = arg1;
|
|
|
|
|
|
|
|
fn(arg2);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
_spdk_reactor_send_msg(spdk_thread_fn fn, void *ctx, void *thread_ctx)
|
|
|
|
{
|
|
|
|
uint32_t core;
|
|
|
|
struct spdk_event *event;
|
|
|
|
|
|
|
|
core = *(uint32_t *)thread_ctx;
|
|
|
|
|
|
|
|
event = spdk_event_allocate(core, _spdk_reactor_msg_passed, fn, ctx);
|
|
|
|
|
|
|
|
spdk_event_call(event);
|
|
|
|
}
|
|
|
|
|
2017-08-17 01:59:06 +00:00
|
|
|
static void
|
|
|
|
get_rusage(void *arg)
|
|
|
|
{
|
|
|
|
struct spdk_reactor *reactor = arg;
|
|
|
|
struct rusage rusage;
|
|
|
|
|
|
|
|
if (getrusage(RUSAGE_THREAD, &rusage) != 0) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (rusage.ru_nvcsw != reactor->rusage.ru_nvcsw || rusage.ru_nivcsw != reactor->rusage.ru_nivcsw) {
|
2017-08-31 03:51:19 +00:00
|
|
|
SPDK_INFOLOG(SPDK_TRACE_REACTOR,
|
|
|
|
"Reactor %d: %ld voluntary context switches and %ld involuntary context switches in the last second.\n",
|
|
|
|
reactor->lcore, rusage.ru_nvcsw - reactor->rusage.ru_nvcsw,
|
|
|
|
rusage.ru_nivcsw - reactor->rusage.ru_nivcsw);
|
2017-08-17 01:59:06 +00:00
|
|
|
}
|
|
|
|
reactor->rusage = rusage;
|
|
|
|
}
|
2017-08-31 03:51:19 +00:00
|
|
|
|
|
|
|
static void
|
|
|
|
_spdk_reactor_context_switch_monitor_start(void *arg1, void *arg2)
|
|
|
|
{
|
|
|
|
struct spdk_reactor *reactor = arg1;
|
|
|
|
|
|
|
|
if (reactor->rusage_poller == NULL) {
|
|
|
|
getrusage(RUSAGE_THREAD, &reactor->rusage);
|
|
|
|
spdk_poller_register(&reactor->rusage_poller, get_rusage, reactor, reactor->lcore, 1000000);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
_spdk_reactor_context_switch_monitor_stop(void *arg1, void *arg2)
|
|
|
|
{
|
|
|
|
struct spdk_reactor *reactor = arg1;
|
|
|
|
|
|
|
|
if (reactor->rusage_poller != NULL) {
|
|
|
|
spdk_poller_unregister(&reactor->rusage_poller, NULL);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
spdk_reactor_enable_context_switch_monitor(bool enable)
|
|
|
|
{
|
|
|
|
struct spdk_reactor *reactor;
|
|
|
|
spdk_event_fn fn;
|
|
|
|
uint32_t core;
|
|
|
|
|
|
|
|
if (enable != g_context_switch_monitor_enabled) {
|
|
|
|
g_context_switch_monitor_enabled = enable;
|
|
|
|
if (enable) {
|
|
|
|
fn = _spdk_reactor_context_switch_monitor_start;
|
|
|
|
} else {
|
|
|
|
fn = _spdk_reactor_context_switch_monitor_stop;
|
|
|
|
}
|
|
|
|
SPDK_ENV_FOREACH_CORE(core) {
|
|
|
|
reactor = spdk_reactor_get(core);
|
|
|
|
spdk_event_call(spdk_event_allocate(core, fn, reactor, NULL));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
spdk_reactor_context_switch_monitor_enabled(void)
|
|
|
|
{
|
|
|
|
return g_context_switch_monitor_enabled;
|
|
|
|
}
|
2017-08-17 01:59:06 +00:00
|
|
|
|
2016-05-24 18:04:20 +00:00
|
|
|
/**
|
2017-04-25 17:35:22 +00:00
|
|
|
*
|
|
|
|
* \brief This is the main function of the reactor thread.
|
|
|
|
*
|
|
|
|
* \code
|
|
|
|
*
|
|
|
|
* while (1)
|
|
|
|
* if (events to run)
|
|
|
|
* dequeue and run a batch of events
|
|
|
|
*
|
|
|
|
* if (active pollers)
|
|
|
|
* run the first poller in the list and move it to the back
|
|
|
|
*
|
|
|
|
* if (first timer poller has expired)
|
|
|
|
* run the first timer poller and reinsert it in the timer list
|
|
|
|
*
|
|
|
|
* if (idle for at least SPDK_REACTOR_SPIN_TIME_US)
|
|
|
|
* sleep until next timer poller is scheduled to expire
|
|
|
|
* \endcode
|
|
|
|
*
|
|
|
|
*/
|
2016-05-24 18:04:20 +00:00
|
|
|
static int
|
|
|
|
_spdk_reactor_run(void *arg)
|
|
|
|
{
|
|
|
|
struct spdk_reactor *reactor = arg;
|
2016-07-28 15:50:50 +00:00
|
|
|
struct spdk_poller *poller;
|
2016-08-16 19:51:25 +00:00
|
|
|
uint32_t event_count;
|
2017-01-20 17:23:30 +00:00
|
|
|
uint64_t idle_started, now;
|
2016-08-16 19:51:25 +00:00
|
|
|
uint64_t spin_cycles, sleep_cycles;
|
|
|
|
uint32_t sleep_us;
|
2017-01-20 17:57:27 +00:00
|
|
|
uint32_t timer_poll_count;
|
2017-08-22 20:06:57 +00:00
|
|
|
char thread_name[32];
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2017-08-22 20:06:57 +00:00
|
|
|
snprintf(thread_name, sizeof(thread_name), "reactor_%u", reactor->lcore);
|
|
|
|
spdk_allocate_thread(_spdk_reactor_send_msg, &reactor->lcore, thread_name);
|
2017-01-20 17:41:21 +00:00
|
|
|
SPDK_NOTICELOG("Reactor started on core %u on socket %u\n", reactor->lcore,
|
|
|
|
reactor->socket_id);
|
2016-08-16 19:51:25 +00:00
|
|
|
|
2016-08-18 19:52:48 +00:00
|
|
|
spin_cycles = SPDK_REACTOR_SPIN_TIME_US * spdk_get_ticks_hz() / 1000000ULL;
|
|
|
|
sleep_cycles = reactor->max_delay_us * spdk_get_ticks_hz() / 1000000ULL;
|
2017-01-20 17:23:30 +00:00
|
|
|
idle_started = 0;
|
2017-01-20 17:57:27 +00:00
|
|
|
timer_poll_count = 0;
|
2017-08-31 03:51:19 +00:00
|
|
|
if (g_context_switch_monitor_enabled) {
|
|
|
|
_spdk_reactor_context_switch_monitor_start(reactor, NULL);
|
|
|
|
}
|
2016-05-24 18:04:20 +00:00
|
|
|
while (1) {
|
2017-01-20 17:23:30 +00:00
|
|
|
bool took_action = false;
|
|
|
|
|
2017-01-20 17:41:21 +00:00
|
|
|
event_count = _spdk_event_queue_run_batch(reactor);
|
2016-08-16 19:51:25 +00:00
|
|
|
if (event_count > 0) {
|
2017-01-20 17:23:30 +00:00
|
|
|
took_action = true;
|
2016-08-16 19:51:25 +00:00
|
|
|
}
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2016-07-28 15:50:50 +00:00
|
|
|
poller = TAILQ_FIRST(&reactor->active_pollers);
|
|
|
|
if (poller) {
|
|
|
|
TAILQ_REMOVE(&reactor->active_pollers, poller, tailq);
|
2016-09-30 21:00:16 +00:00
|
|
|
poller->state = SPDK_POLLER_STATE_RUNNING;
|
2016-05-24 18:04:20 +00:00
|
|
|
poller->fn(poller->arg);
|
2016-09-30 21:00:16 +00:00
|
|
|
if (poller->state == SPDK_POLLER_STATE_UNREGISTERED) {
|
|
|
|
_spdk_poller_unregister_complete(poller);
|
|
|
|
} else {
|
|
|
|
poller->state = SPDK_POLLER_STATE_WAITING;
|
|
|
|
TAILQ_INSERT_TAIL(&reactor->active_pollers, poller, tailq);
|
|
|
|
}
|
2017-01-20 17:23:30 +00:00
|
|
|
took_action = true;
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
2017-01-20 17:57:27 +00:00
|
|
|
if (timer_poll_count >= SPDK_TIMER_POLL_ITERATIONS) {
|
|
|
|
poller = TAILQ_FIRST(&reactor->timer_pollers);
|
|
|
|
if (poller) {
|
|
|
|
now = spdk_get_ticks();
|
|
|
|
|
|
|
|
if (now >= poller->next_run_tick) {
|
|
|
|
TAILQ_REMOVE(&reactor->timer_pollers, poller, tailq);
|
|
|
|
poller->state = SPDK_POLLER_STATE_RUNNING;
|
|
|
|
poller->fn(poller->arg);
|
|
|
|
if (poller->state == SPDK_POLLER_STATE_UNREGISTERED) {
|
|
|
|
_spdk_poller_unregister_complete(poller);
|
|
|
|
} else {
|
|
|
|
poller->state = SPDK_POLLER_STATE_WAITING;
|
|
|
|
spdk_poller_insert_timer(reactor, poller, now);
|
|
|
|
}
|
|
|
|
took_action = true;
|
2016-09-30 21:00:16 +00:00
|
|
|
}
|
2016-07-26 19:56:07 +00:00
|
|
|
}
|
2017-01-20 17:57:27 +00:00
|
|
|
timer_poll_count = 0;
|
|
|
|
} else {
|
|
|
|
timer_poll_count++;
|
2016-07-26 19:56:07 +00:00
|
|
|
}
|
|
|
|
|
2017-01-20 17:23:30 +00:00
|
|
|
if (took_action) {
|
|
|
|
/* We were busy this loop iteration. Reset the idle timer. */
|
|
|
|
idle_started = 0;
|
|
|
|
} else if (idle_started == 0) {
|
|
|
|
/* We were previously busy, but this loop we took no actions. */
|
|
|
|
idle_started = spdk_get_ticks();
|
|
|
|
}
|
|
|
|
|
2016-08-16 19:51:25 +00:00
|
|
|
/* Determine if the thread can sleep */
|
2017-01-20 17:23:30 +00:00
|
|
|
if (sleep_cycles && idle_started) {
|
2016-08-18 19:52:48 +00:00
|
|
|
now = spdk_get_ticks();
|
2017-01-20 17:23:30 +00:00
|
|
|
if (now >= (idle_started + spin_cycles)) {
|
2016-08-16 19:51:25 +00:00
|
|
|
sleep_us = reactor->max_delay_us;
|
|
|
|
|
|
|
|
poller = TAILQ_FIRST(&reactor->timer_pollers);
|
|
|
|
if (poller) {
|
|
|
|
/* There are timers registered, so don't sleep beyond
|
|
|
|
* when the next timer should fire */
|
|
|
|
if (poller->next_run_tick < (now + sleep_cycles)) {
|
|
|
|
if (poller->next_run_tick <= now) {
|
|
|
|
sleep_us = 0;
|
|
|
|
} else {
|
2016-08-18 19:52:48 +00:00
|
|
|
sleep_us = ((poller->next_run_tick - now) * 1000000ULL) / spdk_get_ticks_hz();
|
2016-08-16 19:51:25 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (sleep_us > 0) {
|
|
|
|
usleep(sleep_us);
|
|
|
|
}
|
2017-01-20 17:57:27 +00:00
|
|
|
|
|
|
|
/* After sleeping, always poll for timers */
|
|
|
|
timer_poll_count = SPDK_TIMER_POLL_ITERATIONS;
|
2016-08-16 19:51:25 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-05-24 18:04:20 +00:00
|
|
|
if (g_reactor_state != SPDK_REACTOR_STATE_RUNNING) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-08-31 03:51:19 +00:00
|
|
|
_spdk_reactor_context_switch_monitor_stop(reactor, NULL);
|
2016-09-20 23:04:51 +00:00
|
|
|
spdk_free_thread();
|
2016-05-24 18:04:20 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2016-08-16 19:51:25 +00:00
|
|
|
spdk_reactor_construct(struct spdk_reactor *reactor, uint32_t lcore, uint64_t max_delay_us)
|
2016-05-24 18:04:20 +00:00
|
|
|
{
|
|
|
|
reactor->lcore = lcore;
|
2017-03-27 19:59:40 +00:00
|
|
|
reactor->socket_id = spdk_env_get_socket_id(lcore);
|
2017-01-20 17:41:21 +00:00
|
|
|
assert(reactor->socket_id < SPDK_MAX_SOCKET);
|
2016-08-16 19:51:25 +00:00
|
|
|
reactor->max_delay_us = max_delay_us;
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2016-07-28 15:50:50 +00:00
|
|
|
TAILQ_INIT(&reactor->active_pollers);
|
2016-07-26 19:56:07 +00:00
|
|
|
TAILQ_INIT(&reactor->timer_pollers);
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2017-05-24 12:46:55 +00:00
|
|
|
reactor->events = spdk_ring_create(SPDK_RING_TYPE_MP_SC, 65536, reactor->socket_id);
|
2016-08-18 16:45:50 +00:00
|
|
|
assert(reactor->events != NULL);
|
2017-01-20 17:41:21 +00:00
|
|
|
|
|
|
|
reactor->event_mempool = g_spdk_event_mempool[reactor->socket_id];
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
spdk_app_get_core_count(void)
|
|
|
|
{
|
2017-03-27 19:59:40 +00:00
|
|
|
return spdk_env_get_core_count();
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
2016-08-04 17:18:25 +00:00
|
|
|
uint32_t
|
|
|
|
spdk_app_get_current_core(void)
|
|
|
|
{
|
2017-03-27 19:59:40 +00:00
|
|
|
return spdk_env_get_current_core();
|
2016-08-04 17:18:25 +00:00
|
|
|
}
|
|
|
|
|
2016-05-24 18:04:20 +00:00
|
|
|
int
|
|
|
|
spdk_app_parse_core_mask(const char *mask, uint64_t *cpumask)
|
|
|
|
{
|
2017-06-02 18:00:35 +00:00
|
|
|
uint32_t i;
|
2016-05-24 18:04:20 +00:00
|
|
|
char *end;
|
2017-06-02 18:00:35 +00:00
|
|
|
uint64_t validmask;
|
2016-05-24 18:04:20 +00:00
|
|
|
|
|
|
|
if (mask == NULL || cpumask == NULL) {
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
errno = 0;
|
|
|
|
*cpumask = strtoull(mask, &end, 16);
|
|
|
|
if (*end != '\0' || errno) {
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2017-06-02 18:00:35 +00:00
|
|
|
validmask = 0;
|
|
|
|
SPDK_ENV_FOREACH_CORE(i) {
|
|
|
|
if (i >= 64) {
|
|
|
|
break;
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
2017-06-02 18:00:35 +00:00
|
|
|
validmask |= 1ULL << i;
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
2017-06-02 18:00:35 +00:00
|
|
|
*cpumask &= validmask;
|
|
|
|
|
2016-05-24 18:04:20 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-04-03 22:59:41 +00:00
|
|
|
uint64_t
|
|
|
|
spdk_app_get_core_mask(void)
|
2016-05-24 18:04:20 +00:00
|
|
|
{
|
2017-04-03 22:02:49 +00:00
|
|
|
uint32_t i;
|
2017-04-03 22:59:41 +00:00
|
|
|
uint64_t mask = 0;
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2017-04-03 22:02:49 +00:00
|
|
|
SPDK_ENV_FOREACH_CORE(i) {
|
|
|
|
mask |= 1ULL << i;
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
2017-04-03 22:59:41 +00:00
|
|
|
return mask;
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static uint64_t
|
|
|
|
spdk_reactor_get_socket_mask(void)
|
|
|
|
{
|
2017-04-03 22:02:49 +00:00
|
|
|
uint32_t i;
|
2016-05-24 18:04:20 +00:00
|
|
|
uint32_t socket_id;
|
|
|
|
uint64_t socket_info = 0;
|
|
|
|
|
2017-04-03 22:02:49 +00:00
|
|
|
SPDK_ENV_FOREACH_CORE(i) {
|
2017-04-03 22:59:41 +00:00
|
|
|
socket_id = spdk_env_get_socket_id(i);
|
|
|
|
socket_info |= (1ULL << socket_id);
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return socket_info;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2016-06-06 06:23:42 +00:00
|
|
|
spdk_reactors_start(void)
|
2016-05-24 18:04:20 +00:00
|
|
|
{
|
|
|
|
struct spdk_reactor *reactor;
|
2017-04-03 22:02:49 +00:00
|
|
|
uint32_t i, current_core;
|
2017-06-02 18:40:40 +00:00
|
|
|
int rc;
|
2016-05-24 18:04:20 +00:00
|
|
|
|
|
|
|
g_reactor_state = SPDK_REACTOR_STATE_RUNNING;
|
|
|
|
|
2017-04-03 22:02:49 +00:00
|
|
|
current_core = spdk_env_get_current_core();
|
|
|
|
SPDK_ENV_FOREACH_CORE(i) {
|
|
|
|
if (i != current_core) {
|
|
|
|
reactor = spdk_reactor_get(i);
|
2017-06-02 18:40:40 +00:00
|
|
|
rc = spdk_env_thread_launch_pinned(reactor->lcore, _spdk_reactor_run, reactor);
|
|
|
|
if (rc < 0) {
|
|
|
|
SPDK_ERRLOG("Unable to start reactor thread on core %u\n", reactor->lcore);
|
|
|
|
assert(false);
|
|
|
|
return;
|
|
|
|
}
|
2017-04-03 22:02:49 +00:00
|
|
|
}
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Start the master reactor */
|
2017-04-03 22:02:49 +00:00
|
|
|
reactor = spdk_reactor_get(current_core);
|
2017-06-02 18:40:40 +00:00
|
|
|
_spdk_reactor_run(reactor);
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2017-06-02 18:40:40 +00:00
|
|
|
spdk_env_thread_wait_all();
|
2016-05-24 18:04:20 +00:00
|
|
|
|
|
|
|
g_reactor_state = SPDK_REACTOR_STATE_SHUTDOWN;
|
|
|
|
}
|
|
|
|
|
2016-06-06 06:23:42 +00:00
|
|
|
void spdk_reactors_stop(void)
|
2016-05-24 18:04:20 +00:00
|
|
|
{
|
|
|
|
g_reactor_state = SPDK_REACTOR_STATE_EXITING;
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
2017-04-03 22:59:41 +00:00
|
|
|
spdk_reactors_init(unsigned int max_delay_us)
|
2016-05-24 18:04:20 +00:00
|
|
|
{
|
2017-02-24 23:27:42 +00:00
|
|
|
uint32_t i, j;
|
2016-05-24 18:04:20 +00:00
|
|
|
struct spdk_reactor *reactor;
|
2016-06-28 03:24:36 +00:00
|
|
|
uint64_t socket_mask = 0x0;
|
|
|
|
uint8_t socket_count = 0;
|
|
|
|
char mempool_name[32];
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2016-06-28 03:24:36 +00:00
|
|
|
socket_mask = spdk_reactor_get_socket_mask();
|
|
|
|
printf("Occupied cpu socket mask is 0x%lx\n", socket_mask);
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2016-06-28 03:24:36 +00:00
|
|
|
for (i = 0; i < SPDK_MAX_SOCKET; i++) {
|
|
|
|
if ((1ULL << i) & socket_mask) {
|
|
|
|
socket_count++;
|
|
|
|
}
|
|
|
|
}
|
2017-03-23 22:52:18 +00:00
|
|
|
if (socket_count == 0) {
|
|
|
|
printf("No sockets occupied (internal error)\n");
|
|
|
|
return -1;
|
|
|
|
}
|
2016-06-28 03:24:36 +00:00
|
|
|
|
|
|
|
for (i = 0; i < SPDK_MAX_SOCKET; i++) {
|
|
|
|
if ((1ULL << i) & socket_mask) {
|
2017-06-22 14:57:54 +00:00
|
|
|
snprintf(mempool_name, sizeof(mempool_name), "evtpool%d_%d", i, getpid());
|
2016-09-30 21:14:18 +00:00
|
|
|
g_spdk_event_mempool[i] = spdk_mempool_create(mempool_name,
|
2016-06-28 03:24:36 +00:00
|
|
|
(262144 / socket_count),
|
2017-08-16 17:05:24 +00:00
|
|
|
sizeof(struct spdk_event),
|
|
|
|
SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, i);
|
2016-06-28 03:24:36 +00:00
|
|
|
|
|
|
|
if (g_spdk_event_mempool[i] == NULL) {
|
|
|
|
SPDK_ERRLOG("spdk_event_mempool creation failed on socket %d\n", i);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Instead of failing the operation directly, try to create
|
|
|
|
* the mempool on any available sockets in the case that
|
|
|
|
* memory is not evenly installed on all sockets. If still
|
|
|
|
* fails, free all allocated memory and exits.
|
|
|
|
*/
|
2016-09-30 21:14:18 +00:00
|
|
|
g_spdk_event_mempool[i] = spdk_mempool_create(
|
2016-06-28 03:24:36 +00:00
|
|
|
mempool_name,
|
|
|
|
(262144 / socket_count),
|
2017-08-16 17:05:24 +00:00
|
|
|
sizeof(struct spdk_event),
|
|
|
|
SPDK_MEMPOOL_DEFAULT_CACHE_SIZE,
|
2017-01-25 20:35:40 +00:00
|
|
|
SPDK_ENV_SOCKET_ID_ANY);
|
2016-06-28 03:24:36 +00:00
|
|
|
|
|
|
|
if (g_spdk_event_mempool[i] == NULL) {
|
2017-02-24 23:27:42 +00:00
|
|
|
for (j = i - 1; j < i; j--) {
|
|
|
|
if (g_spdk_event_mempool[j] != NULL) {
|
|
|
|
spdk_mempool_free(g_spdk_event_mempool[j]);
|
|
|
|
}
|
|
|
|
}
|
2016-06-28 03:24:36 +00:00
|
|
|
SPDK_ERRLOG("spdk_event_mempool creation failed\n");
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
}
|
2017-07-21 18:13:32 +00:00
|
|
|
} else {
|
|
|
|
g_spdk_event_mempool[i] = NULL;
|
2016-06-28 03:24:36 +00:00
|
|
|
}
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
2017-04-03 22:02:49 +00:00
|
|
|
SPDK_ENV_FOREACH_CORE(i) {
|
2017-04-03 22:59:41 +00:00
|
|
|
reactor = spdk_reactor_get(i);
|
|
|
|
spdk_reactor_construct(reactor, i, max_delay_us);
|
2017-01-20 17:41:21 +00:00
|
|
|
}
|
|
|
|
|
2016-05-24 18:04:20 +00:00
|
|
|
g_reactor_state = SPDK_REACTOR_STATE_INITIALIZED;
|
|
|
|
|
2017-04-03 22:59:41 +00:00
|
|
|
return 0;
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int
|
2016-06-06 06:23:42 +00:00
|
|
|
spdk_reactors_fini(void)
|
2016-05-24 18:04:20 +00:00
|
|
|
{
|
2017-02-24 23:27:42 +00:00
|
|
|
uint32_t i;
|
|
|
|
struct spdk_reactor *reactor;
|
|
|
|
|
2017-04-03 22:02:49 +00:00
|
|
|
SPDK_ENV_FOREACH_CORE(i) {
|
2017-04-03 22:59:41 +00:00
|
|
|
reactor = spdk_reactor_get(i);
|
|
|
|
if (reactor->events != NULL) {
|
2017-05-09 17:48:16 +00:00
|
|
|
spdk_ring_free(reactor->events);
|
2017-02-24 23:27:42 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < SPDK_MAX_SOCKET; i++) {
|
2017-07-21 18:13:32 +00:00
|
|
|
if (g_spdk_event_mempool[i] != NULL) {
|
2017-02-24 23:27:42 +00:00
|
|
|
spdk_mempool_free(g_spdk_event_mempool[i]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-05-24 18:04:20 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2017-01-04 23:52:03 +00:00
|
|
|
_spdk_poller_register(struct spdk_reactor *reactor, struct spdk_poller *poller)
|
2016-05-24 18:04:20 +00:00
|
|
|
{
|
2016-07-26 19:56:07 +00:00
|
|
|
if (poller->period_ticks) {
|
2016-08-18 19:52:48 +00:00
|
|
|
spdk_poller_insert_timer(reactor, poller, spdk_get_ticks());
|
2016-07-26 19:56:07 +00:00
|
|
|
} else {
|
|
|
|
TAILQ_INSERT_TAIL(&reactor->active_pollers, poller, tailq);
|
|
|
|
}
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
2016-10-11 18:20:49 +00:00
|
|
|
static void
|
2017-01-05 01:19:02 +00:00
|
|
|
_spdk_event_add_poller(void *arg1, void *arg2)
|
2016-10-11 18:20:49 +00:00
|
|
|
{
|
2017-01-05 01:19:02 +00:00
|
|
|
struct spdk_reactor *reactor = arg1;
|
|
|
|
struct spdk_poller *poller = arg2;
|
2016-10-11 18:20:49 +00:00
|
|
|
|
2017-01-04 23:52:03 +00:00
|
|
|
_spdk_poller_register(reactor, poller);
|
2016-10-11 18:20:49 +00:00
|
|
|
}
|
|
|
|
|
2016-08-11 23:00:45 +00:00
|
|
|
void
|
|
|
|
spdk_poller_register(struct spdk_poller **ppoller, spdk_poller_fn fn, void *arg,
|
2017-01-04 23:52:03 +00:00
|
|
|
uint32_t lcore, uint64_t period_microseconds)
|
2016-05-24 18:04:20 +00:00
|
|
|
{
|
2016-08-11 23:00:45 +00:00
|
|
|
struct spdk_poller *poller;
|
2016-05-24 18:04:20 +00:00
|
|
|
struct spdk_reactor *reactor;
|
|
|
|
|
2016-08-11 23:00:45 +00:00
|
|
|
poller = calloc(1, sizeof(*poller));
|
|
|
|
if (poller == NULL) {
|
|
|
|
SPDK_ERRLOG("Poller memory allocation failed\n");
|
|
|
|
abort();
|
|
|
|
}
|
|
|
|
|
|
|
|
poller->lcore = lcore;
|
2016-09-30 21:00:16 +00:00
|
|
|
poller->state = SPDK_POLLER_STATE_WAITING;
|
2016-08-11 23:00:45 +00:00
|
|
|
poller->fn = fn;
|
|
|
|
poller->arg = arg;
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2016-07-26 19:56:07 +00:00
|
|
|
if (period_microseconds) {
|
2016-08-18 19:52:48 +00:00
|
|
|
poller->period_ticks = (spdk_get_ticks_hz() * period_microseconds) / 1000000ULL;
|
2016-07-26 19:56:07 +00:00
|
|
|
} else {
|
|
|
|
poller->period_ticks = 0;
|
|
|
|
}
|
|
|
|
|
2016-08-11 23:00:45 +00:00
|
|
|
if (*ppoller != NULL) {
|
|
|
|
SPDK_ERRLOG("Attempted reuse of poller pointer\n");
|
|
|
|
abort();
|
|
|
|
}
|
|
|
|
|
2017-06-02 18:40:40 +00:00
|
|
|
if (lcore >= SPDK_MAX_REACTORS) {
|
|
|
|
SPDK_ERRLOG("Attempted to use lcore %u which is larger than max lcore %u\n",
|
|
|
|
lcore, SPDK_MAX_REACTORS - 1);
|
2017-03-23 06:12:01 +00:00
|
|
|
abort();
|
|
|
|
}
|
|
|
|
|
2016-08-11 23:00:45 +00:00
|
|
|
*ppoller = poller;
|
|
|
|
reactor = spdk_reactor_get(lcore);
|
2016-10-11 18:20:49 +00:00
|
|
|
|
2017-03-27 19:59:40 +00:00
|
|
|
if (lcore == spdk_env_get_current_core()) {
|
2016-10-11 18:20:49 +00:00
|
|
|
/*
|
|
|
|
* The poller is registered to run on the current core, so call the add function
|
|
|
|
* directly.
|
|
|
|
*/
|
2017-01-04 23:52:03 +00:00
|
|
|
_spdk_poller_register(reactor, poller);
|
2016-10-11 18:20:49 +00:00
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* The poller is registered to run on a different core.
|
|
|
|
* Schedule an event to run on the poller's core that will add the poller.
|
|
|
|
*/
|
2017-01-05 01:40:14 +00:00
|
|
|
spdk_event_call(spdk_event_allocate(lcore, _spdk_event_add_poller, reactor, poller));
|
2016-10-11 18:20:49 +00:00
|
|
|
}
|
2016-07-26 19:56:07 +00:00
|
|
|
}
|
|
|
|
|
2016-05-24 18:04:20 +00:00
|
|
|
static void
|
2016-09-30 21:00:16 +00:00
|
|
|
_spdk_poller_unregister(struct spdk_reactor *reactor, struct spdk_poller *poller,
|
|
|
|
struct spdk_event *next)
|
2016-05-24 18:04:20 +00:00
|
|
|
{
|
2016-09-30 21:00:16 +00:00
|
|
|
assert(poller->lcore == reactor->lcore);
|
2017-03-27 19:59:40 +00:00
|
|
|
assert(poller->lcore == spdk_env_get_current_core());
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2016-09-30 21:00:16 +00:00
|
|
|
poller->unregister_complete_event = next;
|
|
|
|
|
|
|
|
if (poller->state == SPDK_POLLER_STATE_RUNNING) {
|
|
|
|
/*
|
|
|
|
* We are being called from the poller_fn, so set the state to unregistered
|
|
|
|
* and let the reactor loop free the poller.
|
|
|
|
*/
|
|
|
|
poller->state = SPDK_POLLER_STATE_UNREGISTERED;
|
2016-07-26 19:56:07 +00:00
|
|
|
} else {
|
2016-09-30 21:00:16 +00:00
|
|
|
/* Poller is not running currently, so just free it. */
|
|
|
|
if (poller->period_ticks) {
|
|
|
|
TAILQ_REMOVE(&reactor->timer_pollers, poller, tailq);
|
|
|
|
} else {
|
|
|
|
TAILQ_REMOVE(&reactor->active_pollers, poller, tailq);
|
|
|
|
}
|
|
|
|
|
|
|
|
_spdk_poller_unregister_complete(poller);
|
2016-07-26 19:56:07 +00:00
|
|
|
}
|
2016-09-30 21:00:16 +00:00
|
|
|
}
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2016-09-30 21:00:16 +00:00
|
|
|
static void
|
2017-01-05 01:19:02 +00:00
|
|
|
_spdk_event_remove_poller(void *arg1, void *arg2)
|
2016-09-30 21:00:16 +00:00
|
|
|
{
|
2017-01-05 01:19:02 +00:00
|
|
|
struct spdk_poller *poller = arg1;
|
2016-09-30 21:00:16 +00:00
|
|
|
struct spdk_reactor *reactor = spdk_reactor_get(poller->lcore);
|
2017-01-05 01:19:02 +00:00
|
|
|
struct spdk_event *next = arg2;
|
2016-08-11 23:00:45 +00:00
|
|
|
|
2016-09-30 21:00:16 +00:00
|
|
|
_spdk_poller_unregister(reactor, poller, next);
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2016-08-11 23:00:45 +00:00
|
|
|
spdk_poller_unregister(struct spdk_poller **ppoller,
|
2016-05-24 18:04:20 +00:00
|
|
|
struct spdk_event *complete)
|
|
|
|
{
|
2016-08-11 23:00:45 +00:00
|
|
|
struct spdk_poller *poller;
|
2016-09-30 21:00:16 +00:00
|
|
|
uint32_t lcore;
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2016-08-11 23:00:45 +00:00
|
|
|
poller = *ppoller;
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2016-08-11 23:00:45 +00:00
|
|
|
*ppoller = NULL;
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2016-08-11 23:00:45 +00:00
|
|
|
if (poller == NULL) {
|
|
|
|
if (complete) {
|
|
|
|
spdk_event_call(complete);
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
2016-05-24 18:04:20 +00:00
|
|
|
|
2016-09-30 21:00:16 +00:00
|
|
|
lcore = poller->lcore;
|
|
|
|
|
2017-03-27 19:59:40 +00:00
|
|
|
if (lcore == spdk_env_get_current_core()) {
|
2016-09-30 21:00:16 +00:00
|
|
|
/*
|
|
|
|
* The poller is registered on the current core, so call the remove function
|
|
|
|
* directly.
|
|
|
|
*/
|
|
|
|
_spdk_poller_unregister(spdk_reactor_get(lcore), poller, complete);
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* The poller is registered on a different core.
|
|
|
|
* Schedule an event to run on the poller's core that will remove the poller.
|
|
|
|
*/
|
2017-01-05 01:40:14 +00:00
|
|
|
spdk_event_call(spdk_event_allocate(lcore, _spdk_event_remove_poller, poller, complete));
|
2016-09-30 21:00:16 +00:00
|
|
|
}
|
2016-05-24 18:04:20 +00:00
|
|
|
}
|
2017-08-17 01:59:06 +00:00
|
|
|
|
|
|
|
SPDK_LOG_REGISTER_TRACE_FLAG("reactor", SPDK_TRACE_REACTOR)
|