d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

thread: reap unreg pollers in intr

Browse Source 
When intr mode is enabled, it will be common that
poller is unregistered during interrupt processing.
Since poller unregister is a delayed operation,
mark it in spdk_thread object, and reap unregistered
pollers out of poller execution.

Fixes #2143

Change-Id: Ieb61fc7685f85af5c15e833dd1dd56f8c97a3b12
Signed-off-by: Liu Xiaodong <xiaodong.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5770
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: Tomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: Ben Walker <benjamin.walker@intel.com>
Tested-by: SPDK CI Jenkins <sys_sgci@intel.com>
This commit is contained in:
Liu Xiaodong 2021-09-03 19:22:35 -04:00 committed by Jim Harris
parent 3c271d5e4c
commit b1906912e9
3 changed files with 73 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
autotest.sh
View File

@ -188,6 +188,7 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then
run_test "bdevperf_config" test/bdev/bdevperf/test_config.sh
if [[ $(uname -s) == Linux ]]; then
run_test "reactor_set_interrupt" test/interrupt/reactor_set_interrupt.sh
run_test "reap_unregistered_poller" test/interrupt/reap_unregistered_poller.sh
fi
fi

38
lib/thread/thread.c
View File

@ -94,6 +94,7 @@ struct spdk_poller {
spdk_poller_fn fn;
void *arg;
struct spdk_thread *thread;
/* Native interruptfd for period or busy poller */
int interruptfd;
spdk_poller_set_interrupt_mode_cb set_intr_cb_fn;
void *set_intr_cb_arg;
@ -153,6 +154,7 @@ struct spdk_thread {
/* Indicates whether this spdk_thread currently runs in interrupt. */
bool in_interrupt;
bool poller_unregistered;
struct spdk_fd_group *fgrp;
/* User context allocated at the end */
@ -737,7 +739,6 @@ poller_insert_timer(struct spdk_thread *thread, struct spdk_poller *poller, uint
}
}
#ifdef __linux__
static inline void
poller_remove_timer(struct spdk_thread *thread, struct spdk_poller *poller)
{
@ -753,7 +754,6 @@ poller_remove_timer(struct spdk_thread *thread, struct spdk_poller *poller)
thread->first_timed_poller = RB_MIN(timed_pollers_tree, &thread->timed_pollers);
}
}
#endif
static void
thread_insert_poller(struct spdk_thread *thread, struct spdk_poller *poller)
@ -996,6 +996,28 @@ spdk_thread_poll(struct spdk_thread *thread, uint32_t max_msgs, uint64_t now)
SPDK_ERRLOG("failed to acknowledge msg queue: %s.\n", spdk_strerror(errno));
}
}
/* Reap unregistered pollers out of poller execution in intr mode */
if (spdk_unlikely(thread->poller_unregistered)) {
struct spdk_poller *poller, *tmp;
TAILQ_FOREACH_REVERSE_SAFE(poller, &thread->active_pollers,
active_pollers_head, tailq, tmp) {
if (poller->state == SPDK_POLLER_STATE_UNREGISTERED) {
TAILQ_REMOVE(&thread->active_pollers, poller, tailq);
free(poller);
}
}
RB_FOREACH_SAFE(poller, timed_pollers_tree, &thread->timed_pollers, tmp) {
if (poller->state == SPDK_POLLER_STATE_UNREGISTERED) {
poller_remove_timer(thread, poller);
free(poller);
}
}
thread->poller_unregistered = false;
}
}
@ -1582,8 +1604,16 @@ spdk_poller_unregister(struct spdk_poller **ppoller)
return;
}
if (spdk_interrupt_mode_is_enabled() && poller->interruptfd >= 0) {
poller_interrupt_fini(poller);
if (spdk_interrupt_mode_is_enabled()) {
/* Release the interrupt resource for period or busy poller */
if (poller->interruptfd >= 0) {
poller_interrupt_fini(poller);
}
/* Mark there is poller unregistered. Then unregistered pollers will
* get reaped by spdk_thread_poll also in intr mode.
*/
thread->poller_unregistered = true;
}
/* If the poller was paused, put it on the active_pollers list so that

38
test/interrupt/reap_unregistered_poller.sh Executable file
View File

@ -0,0 +1,38 @@
#!/usr/bin/env bash
testdir=$(readlink -f $(dirname $0))
rootdir=$(readlink -f $testdir/../..)
source $rootdir/test/common/autotest_common.sh
source $testdir/interrupt_common.sh
export PYTHONPATH=$rootdir/examples/interrupt_tgt
# Set reactors with intr_tgt in intr mode
start_intr_tgt
# Record names of native created pollers.
app_thread=$(rpc_cmd thread_get_pollers | jq -r '.threads[0]')
native_pollers=$(jq -r '.active_pollers[].name' <<< $app_thread)
native_pollers+=" "
native_pollers+=$(jq -r '.timed_pollers[].name' <<< $app_thread)
# Create one aio_bdev.
# During the creation, vbdev examine process will get bdev_aio create
# pollers like bdev_aio_group_poll poller, and then unregister it.
setup_bdev_aio
# Record names of remaining pollers.
app_thread=$(rpc_cmd thread_get_pollers | jq -r '.threads[0]')
remaining_pollers=$(jq -r '.active_pollers[].name' <<< $app_thread)
remaining_pollers+=" "
remaining_pollers+=$(jq -r '.timed_pollers[].name' <<< $app_thread)
# Since bdev_aio created pollers were already unregistered, so
# remaining_pollers should be same with native_pollers.
if [[ "$remaining_pollers" != "$native_pollers" ]]; then
return 1
fi
trap - SIGINT SIGTERM EXIT
killprocess $intr_tgt_pid
cleanup