d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

examples/performance-thread: add lthread subsystem

Browse Source 
This commit adds the lightweight thread subsystem used by the
performance-thread sample applications.

Signed-off-by: Ian Betts <ian.betts@intel.com>
Acked-by: Tomasz Kulasek <tomaszx.kulasek@intel.com>
This commit is contained in:
Ian Betts 2015-12-08 06:05:15 +00:00 committed by Thomas Monjalon
parent 4d1a771bd8
commit 116819b9ed
22 changed files with 5195 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
examples/performance-thread/common/arch/x86/ctx.c Normal file
View File

@ -0,0 +1,93 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* https://github.com/halayli/lthread which carries the following license.
*
* Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#if defined(__x86_64__)
__asm__ (
".text\n"
".p2align 4,,15\n"
".globl ctx_switch\n"
".globl _ctx_switch\n"
"ctx_switch:\n"
"_ctx_switch:\n"
" movq %rsp, 0(%rsi) # save stack_pointer\n"
" movq %rbp, 8(%rsi) # save frame_pointer\n"
" movq (%rsp), %rax # save insn_pointer\n"
" movq %rax, 16(%rsi)\n"
" movq %rbx, 24(%rsi)\n # save rbx,r12-r15\n"
" movq 24(%rdi), %rbx\n"
" movq %r15, 56(%rsi)\n"
" movq %r14, 48(%rsi)\n"
" movq 48(%rdi), %r14\n"
" movq 56(%rdi), %r15\n"
" movq %r13, 40(%rsi)\n"
" movq %r12, 32(%rsi)\n"
" movq 32(%rdi), %r12\n"
" movq 40(%rdi), %r13\n"
" movq 0(%rdi), %rsp # restore stack_pointer\n"
" movq 16(%rdi), %rax # restore insn_pointer\n"
" movq 8(%rdi), %rbp # restore frame_pointer\n"
" movq %rax, (%rsp)\n"
" ret\n"
);
#else
#pragma GCC error "__x86_64__ is not defined"
#endif

57
examples/performance-thread/common/arch/x86/ctx.h Normal file
View File

@ -0,0 +1,57 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef CTX_H
#define CTX_H
/*
* CPU context registers
*/
struct ctx {
void *rsp; /* 0 */
void *rbp; /* 8 */
void *rip; /* 16 */
void *rbx; /* 24 */
void *r12; /* 32 */
void *r13; /* 40 */
void *r14; /* 48 */
void *r15; /* 56 */
};
void
ctx_switch(struct ctx *new_ctx, struct ctx *curr_ctx);
#endif /* RTE_CTX_H_ */

42
examples/performance-thread/common/common.mk Normal file
View File

@ -0,0 +1,42 @@
#
# BSD LICENSE
#
# Copyright(c) 2015 Intel Corporation. All rights reserved.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# list the C files belonhing to the lthread subsystem, these are common to all lthread apps
SRCS-y += ../common/lthread.c \
../common/lthread_sched.c \
../common/lthread_cond.c \
../common/lthread_tls.c \
../common/lthread_mutex.c \
../common/lthread_diag.c \
../common/arch/x86/ctx.c
INCLUDES += -I$(RTE_SDK)/examples/performance-thread/common/ -I$(RTE_SDK)/examples/performance-thread/common/arch/x86/

530
examples/performance-thread/common/lthread.c Normal file
View File

@ -0,0 +1,530 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Some portions of this software is derived from the
* https://github.com/halayli/lthread which carrys the following license.
*
* Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#define RTE_MEM 1
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <stddef.h>
#include <limits.h>
#include <inttypes.h>
#include <unistd.h>
#include <pthread.h>
#include <fcntl.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <rte_config.h>
#include <rte_log.h>
#include <ctx.h>
#include "lthread_api.h"
#include "lthread.h"
#include "lthread_timer.h"
#include "lthread_tls.h"
#include "lthread_objcache.h"
#include "lthread_diag.h"
/*
* This function gets called after an lthread function has returned.
*/
void _lthread_exit_handler(struct lthread *lt)
{
lt->state |= BIT(ST_LT_EXITED);
if (!(lt->state & BIT(ST_LT_DETACH))) {
/* thread is this not explicitly detached
* it must be joinable, so we call lthread_exit().
*/
lthread_exit(NULL);
}
/* if we get here the thread is detached so we can reschedule it,
* allowing the scheduler to free it
*/
_reschedule();
}
/*
* Free resources allocated to an lthread
*/
void _lthread_free(struct lthread *lt)
{
DIAG_EVENT(lt, LT_DIAG_LTHREAD_FREE, lt, 0);
/* invoke any user TLS destructor functions */
_lthread_tls_destroy(lt);
/* free memory allocated for TLS defined using RTE_PER_LTHREAD macros */
if (sizeof(void *) < (uint64_t)RTE_PER_LTHREAD_SECTION_SIZE)
_lthread_objcache_free(lt->tls->root_sched->per_lthread_cache,
lt->per_lthread_data);
/* free pthread style TLS memory */
_lthread_objcache_free(lt->tls->root_sched->tls_cache, lt->tls);
/* free the stack */
_lthread_objcache_free(lt->stack_container->root_sched->stack_cache,
lt->stack_container);
/* now free the thread */
_lthread_objcache_free(lt->root_sched->lthread_cache, lt);
}
/*
* Allocate a stack and maintain a cache of stacks
*/
struct lthread_stack *_stack_alloc(void)
{
struct lthread_stack *s;
s = _lthread_objcache_alloc((THIS_SCHED)->stack_cache);
LTHREAD_ASSERT(s != NULL);
s->root_sched = THIS_SCHED;
s->stack_size = LTHREAD_MAX_STACK_SIZE;
return s;
}
/*
* Execute a ctx by invoking the start function
* On return call an exit handler if the user has provided one
*/
static void _lthread_exec(void *arg)
{
struct lthread *lt = (struct lthread *)arg;
/* invoke the contexts function */
lt->fun(lt->arg);
/* do exit handling */
if (lt->exit_handler != NULL)
lt->exit_handler(lt);
}
/*
* Initialize an lthread
* Set its function, args, and exit handler
*/
void
_lthread_init(struct lthread *lt,
lthread_func_t fun, void *arg, lthread_exit_func exit_handler)
{
/* set ctx func and args */
lt->fun = fun;
lt->arg = arg;
lt->exit_handler = exit_handler;
/* set initial state */
lt->birth = _sched_now();
lt->state = BIT(ST_LT_INIT);
lt->join = LT_JOIN_INITIAL;
}
/*
* set the lthread stack
*/
void _lthread_set_stack(struct lthread *lt, void *stack, size_t stack_size)
{
char *stack_top = (char *)stack + stack_size;
void **s = (void **)stack_top;
/* set stack */
lt->stack = stack;
lt->stack_size = stack_size;
/* set initial context */
s[-3] = NULL;
s[-2] = (void *)lt;
lt->ctx.rsp = (void *)(stack_top - (4 * sizeof(void *)));
lt->ctx.rbp = (void *)(stack_top - (3 * sizeof(void *)));
lt->ctx.rip = (void *)_lthread_exec;
}
/*
* Create an lthread on the current scheduler
* If there is no current scheduler on this pthread then first create one
*/
int
lthread_create(struct lthread **new_lt, int lcore_id,
lthread_func_t fun, void *arg)
{
if ((new_lt == NULL) || (fun == NULL))
return POSIX_ERRNO(EINVAL);
if (lcore_id < 0)
lcore_id = rte_lcore_id();
else if (lcore_id > LTHREAD_MAX_LCORES)
return POSIX_ERRNO(EINVAL);
struct lthread *lt = NULL;
if (THIS_SCHED == NULL) {
THIS_SCHED = _lthread_sched_create(0);
if (THIS_SCHED == NULL) {
perror("Failed to create scheduler");
return POSIX_ERRNO(EAGAIN);
}
}
/* allocate a thread structure */
lt = _lthread_objcache_alloc((THIS_SCHED)->lthread_cache);
if (lt == NULL)
return POSIX_ERRNO(EAGAIN);
bzero(lt, sizeof(struct lthread));
lt->root_sched = THIS_SCHED;
/* set the function args and exit handlder */
_lthread_init(lt, fun, arg, _lthread_exit_handler);
/* put it in the ready queue */
*new_lt = lt;
if (lcore_id < 0)
lcore_id = rte_lcore_id();
DIAG_CREATE_EVENT(lt, LT_DIAG_LTHREAD_CREATE);
rte_wmb();
_ready_queue_insert(_lthread_sched_get(lcore_id), lt);
return 0;
}
/*
* Schedules lthread to sleep for `nsecs`
* setting the lthread state to LT_ST_SLEEPING.
* lthread state is cleared upon resumption or expiry.
*/
static inline void _lthread_sched_sleep(struct lthread *lt, uint64_t nsecs)
{
uint64_t state = lt->state;
uint64_t clks = _ns_to_clks(nsecs);
if (clks) {
_timer_start(lt, clks);
lt->state = state | BIT(ST_LT_SLEEPING);
}
DIAG_EVENT(lt, LT_DIAG_LTHREAD_SLEEP, clks, 0);
_suspend();
}
/*
* Cancels any running timer.
* This can be called multiple times on the same lthread regardless if it was
* sleeping or not.
*/
int _lthread_desched_sleep(struct lthread *lt)
{
uint64_t state = lt->state;
if (state & BIT(ST_LT_SLEEPING)) {
_timer_stop(lt);
state &= (CLEARBIT(ST_LT_SLEEPING) & CLEARBIT(ST_LT_EXPIRED));
lt->state = state | BIT(ST_LT_READY);
return 1;
}
return 0;
}
/*
* set user data pointer in an lthread
*/
void lthread_set_data(void *data)
{
if (sizeof(void *) == RTE_PER_LTHREAD_SECTION_SIZE)
THIS_LTHREAD->per_lthread_data = data;
}
/*
* Retrieve user data pointer from an lthread
*/
void *lthread_get_data(void)
{
return THIS_LTHREAD->per_lthread_data;
}
/*
* Return the current lthread handle
*/
struct lthread *lthread_current(void)
{
struct lthread_sched *sched = THIS_SCHED;
if (sched)
return sched->current_lthread;
return NULL;
}
/*
* Tasklet to cancel a thread
*/
static void
_cancel(void *arg)
{
struct lthread *lt = (struct lthread *) arg;
lt->state |= BIT(ST_LT_CANCELLED);
lthread_detach();
}
/*
* Mark the specified as canceled
*/
int lthread_cancel(struct lthread *cancel_lt)
{
struct lthread *lt;
if ((cancel_lt == NULL) || (cancel_lt == THIS_LTHREAD))
return POSIX_ERRNO(EINVAL);
DIAG_EVENT(cancel_lt, LT_DIAG_LTHREAD_CANCEL, cancel_lt, 0);
if (cancel_lt->sched != THIS_SCHED) {
/* spawn task-let to cancel the thread */
lthread_create(&lt,
cancel_lt->sched->lcore_id,
_cancel,
cancel_lt);
return 0;
}
cancel_lt->state |= BIT(ST_LT_CANCELLED);
return 0;
}
/*
* Suspend the current lthread for specified time
*/
void lthread_sleep(uint64_t nsecs)
{
struct lthread *lt = THIS_LTHREAD;
_lthread_sched_sleep(lt, nsecs);
}
/*
* Suspend the current lthread for specified time
*/
void lthread_sleep_clks(uint64_t clks)
{
struct lthread *lt = THIS_LTHREAD;
uint64_t state = lt->state;
if (clks) {
_timer_start(lt, clks);
lt->state = state | BIT(ST_LT_SLEEPING);
}
DIAG_EVENT(lt, LT_DIAG_LTHREAD_SLEEP, clks, 0);
_suspend();
}
/*
* Requeue the current thread to the back of the ready queue
*/
void lthread_yield(void)
{
struct lthread *lt = THIS_LTHREAD;
DIAG_EVENT(lt, LT_DIAG_LTHREAD_YIELD, 0, 0);
_ready_queue_insert(THIS_SCHED, lt);
ctx_switch(&(THIS_SCHED)->ctx, &lt->ctx);
}
/*
* Exit the current lthread
* If a thread is joining pass the user pointer to it
*/
void lthread_exit(void *ptr)
{
struct lthread *lt = THIS_LTHREAD;
/* if thread is detached (this is not valid) just exit */
if (lt->state & BIT(ST_LT_DETACH))
return;
/* There is a race between lthread_join() and lthread_exit()
* - if exit before join then we suspend and resume on join
* - if join before exit then we resume the joining thread
*/
if ((lt->join == LT_JOIN_INITIAL)
&& rte_atomic64_cmpset(&lt->join, LT_JOIN_INITIAL,
LT_JOIN_EXITING)) {
DIAG_EVENT(lt, LT_DIAG_LTHREAD_EXIT, 1, 0);
_suspend();
/* set the exit value */
if ((ptr != NULL) && (lt->lt_join->lt_exit_ptr != NULL))
*(lt->lt_join->lt_exit_ptr) = ptr;
/* let the joining thread know we have set the exit value */
lt->join = LT_JOIN_EXIT_VAL_SET;
} else {
DIAG_EVENT(lt, LT_DIAG_LTHREAD_EXIT, 0, 0);
/* set the exit value */
if ((ptr != NULL) && (lt->lt_join->lt_exit_ptr != NULL))
*(lt->lt_join->lt_exit_ptr) = ptr;
/* let the joining thread know we have set the exit value */
lt->join = LT_JOIN_EXIT_VAL_SET;
_ready_queue_insert(lt->lt_join->sched,
(struct lthread *)lt->lt_join);
}
/* wait until the joinging thread has collected the exit value */
while (lt->join != LT_JOIN_EXIT_VAL_READ)
_reschedule();
/* reset join state */
lt->join = LT_JOIN_INITIAL;
/* detach it so its resources can be released */
lt->state |= (BIT(ST_LT_DETACH) | BIT(ST_LT_EXITED));
}
/*
* Join an lthread
* Suspend until the joined thread returns
*/
int lthread_join(struct lthread *lt, void **ptr)
{
if (lt == NULL)
return POSIX_ERRNO(EINVAL);
struct lthread *current = THIS_LTHREAD;
uint64_t lt_state = lt->state;
/* invalid to join a detached thread, or a thread that is joined */
if ((lt_state & BIT(ST_LT_DETACH)) || (lt->join == LT_JOIN_THREAD_SET))
return POSIX_ERRNO(EINVAL);
/* pointer to the joining thread and a poingter to return a value */
lt->lt_join = current;
current->lt_exit_ptr = ptr;
/* There is a race between lthread_join() and lthread_exit()
* - if join before exit we suspend and will resume when exit is called
* - if exit before join we resume the exiting thread
*/
if ((lt->join == LT_JOIN_INITIAL)
&& rte_atomic64_cmpset(&lt->join, LT_JOIN_INITIAL,
LT_JOIN_THREAD_SET)) {
DIAG_EVENT(current, LT_DIAG_LTHREAD_JOIN, lt, 1);
_suspend();
} else {
DIAG_EVENT(current, LT_DIAG_LTHREAD_JOIN, lt, 0);
_ready_queue_insert(lt->sched, lt);
}
/* wait for exiting thread to set return value */
while (lt->join != LT_JOIN_EXIT_VAL_SET)
_reschedule();
/* collect the return value */
if (ptr != NULL)
*ptr = *current->lt_exit_ptr;
/* let the exiting thread proceed to exit */
lt->join = LT_JOIN_EXIT_VAL_READ;
return 0;
}
/*
* Detach current lthread
* A detached thread cannot be joined
*/
void lthread_detach(void)
{
struct lthread *lt = THIS_LTHREAD;
DIAG_EVENT(lt, LT_DIAG_LTHREAD_DETACH, 0, 0);
uint64_t state = lt->state;
lt->state = state | BIT(ST_LT_DETACH);
}
/*
* Set function name of an lthread
* this is a debug aid
*/
void lthread_set_funcname(const char *f)
{
struct lthread *lt = THIS_LTHREAD;
strncpy(lt->funcname, f, sizeof(lt->funcname));
lt->funcname[sizeof(lt->funcname)-1] = 0;
}

99
examples/performance-thread/common/lthread.h Normal file
View File

@ -0,0 +1,99 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Some portions of this software is derived from the
* https://github.com/halayli/lthread which carrys the following license.
*
* Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef LTHREAD_H_
#define LTHREAD_H_
#include <rte_per_lcore.h>
#include "lthread_api.h"
#include "lthread_diag.h"
struct lthread;
struct lthread_sched;
/* function to be called when a context function returns */
typedef void (*lthread_exit_func) (struct lthread *);
void _lthread_exit_handler(struct lthread *lt);
void lthread_set_funcname(const char *f);
void _lthread_sched_busy_sleep(struct lthread *lt, uint64_t nsecs);
int _lthread_desched_sleep(struct lthread *lt);
void _lthread_free(struct lthread *lt);
struct lthread_sched *_lthread_sched_get(int lcore_id);
struct lthread_stack *_stack_alloc(void);
struct
lthread_sched *_lthread_sched_create(size_t stack_size);
void
_lthread_init(struct lthread *lt,
lthread_func_t fun, void *arg, lthread_exit_func exit_handler);
void _lthread_set_stack(struct lthread *lt, void *stack, size_t stack_size);
#endif /* LTHREAD_H_ */

829
examples/performance-thread/common/lthread_api.h Normal file
View File

@ -0,0 +1,829 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Some portions of this software may have been derived from the
* https://github.com/halayli/lthread which carrys the following license.
*
* Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/**
* @file
*
* This file contains the public API for the L-thread subsystem
*
* The L_thread subsystem provides a simple cooperative scheduler to
* enable arbitrary functions to run as cooperative threads within a
* single P-thread.
*
* The subsystem provides a P-thread like API that is intended to assist in
* reuse of legacy code written for POSIX p_threads.
*
* The L-thread subsystem relies on cooperative multitasking, as such
* an L-thread must possess frequent rescheduling points. Often these
* rescheduling points are provided transparently when the application
* invokes an L-thread API.
*
* In some applications it is possible that the program may enter a loop the
* exit condition for which depends on the action of another thread or a
* response from hardware. In such a case it is necessary to yield the thread
* periodically in the loop body, to allow other threads an opportunity to
* run. This can be done by inserting a call to lthread_yield() or
* lthread_sleep(n) in the body of the loop.
*
* If the application makes expensive / blocking system calls or does other
* work that would take an inordinate amount of time to complete, this will
* stall the cooperative scheduler resulting in very poor performance.
*
* In such cases an L-thread can be migrated temporarily to another scheduler
* running in a different P-thread on another core. When the expensive or
* blocking operation is completed it can be migrated back to the original
* scheduler. In this way other threads can continue to run on the original
* scheduler and will be completely unaffected by the blocking behaviour.
* To migrate an L-thread to another scheduler the API lthread_set_affinity()
* is provided.
*
* If L-threads that share data are running on the same core it is possible
* to design programs where mutual exclusion mechanisms to protect shared data
* can be avoided. This is due to the fact that the cooperative threads cannot
* preempt each other.
*
* There are two cases where mutual exclusion mechanisms are necessary.
*
* a) Where the L-threads sharing data are running on different cores.
* b) Where code must yield while updating data shared with another thread.
*
* The L-thread subsystem provides a set of mutex APIs to help with such
* scenarios, however excessive reliance on on these will impact performance
* and is best avoided if possible.
*
* L-threads can synchronise using a fast condition variable implementation
* that supports signal and broadcast. An L-thread running on any core can
* wait on a condition.
*
* L-threads can have L-thread local storage with an API modelled on either the
* P-thread get/set specific API or using PER_LTHREAD macros modelled on the
* RTE_PER_LCORE macros. Alternatively a simple user data pointer may be set
* and retrieved from a thread.
*/
#ifndef LTHREAD_H
#define LTHREAD_H
#include <stdint.h>
#include <sys/socket.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <rte_cycles.h>
struct lthread;
struct lthread_cond;
struct lthread_mutex;
struct lthread_condattr;
struct lthread_mutexattr;
typedef void (*lthread_func_t) (void *);
/*
* Define the size of stack for an lthread
* Then this is the size that will be allocated on lthread creation
* This is a fixed size and will not grow.
*/
#define LTHREAD_MAX_STACK_SIZE (1024*64)
/**
* Define the maximum number of TLS keys that can be created
*
*/
#define LTHREAD_MAX_KEYS 1024
/**
* Define the maximum number of attempts to destroy an lthread's
* TLS data on thread exit
*/
#define LTHREAD_DESTRUCTOR_ITERATIONS 4
/**
* Define the maximum number of lcores that will support lthreads
*/
#define LTHREAD_MAX_LCORES RTE_MAX_LCORE
/**
* How many lthread objects to pre-allocate as the system grows
* applies to lthreads + stacks, TLS, mutexs, cond vars.
*
* @see _lthread_alloc()
* @see _cond_alloc()
* @see _mutex_alloc()
*
*/
#define LTHREAD_PREALLOC 100
/**
* Set the number of schedulers in the system.
*
* This function may optionally be called before starting schedulers.
*
* If the number of schedulers is not set, or set to 0 then each scheduler
* will begin scheduling lthreads immediately it is started.
* If the number of schedulers is set to greater than 0, then each scheduler
* will wait until all schedulers have started before beginning to schedule
* lthreads.
*
* If an application wishes to have threads migrate between cores using
* lthread_set_affinity(), or join threads running on other cores using
* lthread_join(), then it is prudent to set the number of schedulers to ensure
* that all schedulers are initialised beforehand.
*
* @param num
* the number of schedulers in the system
* @return
* the number of schedulers in the system
*/
int lthread_num_schedulers_set(int num);
/**
* Return the number of schedulers currently running
* @return
* the number of schedulers in the system
*/
int lthread_active_schedulers(void);
/**
* Shutdown the specified scheduler
*
* This function tells the specified scheduler to
* exit if/when there is no more work to do.
*
* Note that although the scheduler will stop
* resources are not freed.
*
* @param lcore
* The lcore of the scheduler to shutdown
*
* @return
* none
*/
void lthread_scheduler_shutdown(unsigned lcore);
/**
* Shutdown all schedulers
*
* This function tells all schedulers including the current scheduler to
* exit if/when there is no more work to do.
*
* Note that although the schedulers will stop
* resources are not freed.
*
* @return
* none
*/
void lthread_scheduler_shutdown_all(void);
/**
* Run the lthread scheduler
*
* Runs the lthread scheduler.
* This function returns only if/when all lthreads have exited.
* This function must be the main loop of an EAL thread.
*
* @return
* none
*/
void lthread_run(void);
/**
* Create an lthread
*
* Creates an lthread and places it in the ready queue on a particular
* lcore.
*
* If no scheduler exists yet on the curret lcore then one is created.
*
* @param new_lt
* Pointer to an lthread pointer that will be initialized
* @param lcore
* the lcore the thread should be started on or the current clore
* -1 the current lcore
* 0 - LTHREAD_MAX_LCORES any other lcore
* @param lthread_func
* Pointer to the function the for the thread to run
* @param arg
* Pointer to args that will be passed to the thread
*
* @return
* 0 success
* EAGAIN no resources available
* EINVAL NULL thread or function pointer, or lcore_id out of range
*/
int
lthread_create(struct lthread **new_lt,
int lcore, lthread_func_t func, void *arg);
/**
* Cancel an lthread
*
* Cancels an lthread and causes it to be terminated
* If the lthread is detached it will be freed immediately
* otherwise its resources will not be released until it is joined.
*
* @param new_lt
* Pointer to an lthread that will be cancelled
*
* @return
* 0 success
* EINVAL thread was NULL
*/
int lthread_cancel(struct lthread *lt);
/**
* Join an lthread
*
* Joins the current thread with the specified lthread, and waits for that
* thread to exit.
* Passes an optional pointer to collect returned data.
*
* @param lt
* Pointer to the lthread to be joined
* @param ptr
* Pointer to pointer to collect returned data
*
0 * @return
* 0 success
* EINVAL lthread could not be joined.
*/
int lthread_join(struct lthread *lt, void **ptr);
/**
* Detach an lthread
*
* Detaches the current thread
* On exit a detached lthread will be freed immediately and will not wait
* to be joined. The default state for a thread is not detached.
*
* @return
* none
*/
void lthread_detach(void);
/**
* Exit an lthread
*
* Terminate the current thread, optionally return data.
* The data may be collected by lthread_join()
*
* After calling this function the lthread will be suspended until it is
* joined. After it is joined then its resources will be freed.
*
* @param ptr
* Pointer to pointer to data to be returned
*
* @return
* none
*/
void lthread_exit(void *val);
/**
* Cause the current lthread to sleep for n nanoseconds
*
* The current thread will be suspended until the specified time has elapsed
* or has been exceeded.
*
* Execution will switch to the next lthread that is ready to run
*
* @param nsecs
* Number of nanoseconds to sleep
*
* @return
* none
*/
void lthread_sleep(uint64_t nsecs);
/**
* Cause the current lthread to sleep for n cpu clock ticks
*
* The current thread will be suspended until the specified time has elapsed
* or has been exceeded.
*
* Execution will switch to the next lthread that is ready to run
*
* @param clks
* Number of clock ticks to sleep
*
* @return
* none
*/
void lthread_sleep_clks(uint64_t clks);
/**
* Yield the current lthread
*
* The current thread will yield and execution will switch to the
* next lthread that is ready to run
*
* @return
* none
*/
void lthread_yield(void);
/**
* Migrate the current thread to another scheduler
*
* This function migrates the current thread to another scheduler.
* Execution will switch to the next lthread that is ready to run on the
* current scheduler. The current thread will be resumed on the new scheduler.
*
* @param lcore
* The lcore to migrate to
*
* @return
* 0 success we are now running on the specified core
* EINVAL the destination lcore was not valid
*/
int lthread_set_affinity(unsigned lcore);
/**
* Return the current lthread
*
* Returns the current lthread
*
* @return
* pointer to the current lthread
*/
struct lthread
*lthread_current(void);
/**
* Associate user data with an lthread
*
* This function sets a user data pointer in the current lthread
* The pointer can be retrieved with lthread_get_data()
* It is the users responsibility to allocate and free any data referenced
* by the user pointer.
*
* @param data
* pointer to user data
*
* @return
* none
*/
void lthread_set_data(void *data);
/**
* Get user data for the current lthread
*
* This function returns a user data pointer for the current lthread
* The pointer must first be set with lthread_set_data()
* It is the users responsibility to allocate and free any data referenced
* by the user pointer.
*
* @return
* pointer to user data
*/
void
*lthread_get_data(void);
struct lthread_key;
typedef void (*tls_destructor_func) (void *);
/**
* Create a key for lthread TLS
*
* This function is modelled on pthread_key_create
* It creates a thread-specific data key visible to all lthreads on the
* current scheduler.
*
* Key values may be used to locate thread-specific data.
* The same key value may be used by different threads, the values bound
* to the key by lthread_setspecific() are maintained on a per-thread
* basis and persist for the life of the calling thread.
*
* An optional destructor function may be associated with each key value.
* At thread exit, if a key value has a non-NULL destructor pointer, and the
* thread has a non-NULL value associated with the key, the function pointed
* to is called with the current associated value as its sole argument.
*
* @param key
* Pointer to the key to be created
* @param destructor
* Pointer to destructor function
*
* @return
* 0 success
* EINVAL the key ptr was NULL
* EAGAIN no resources available
*/
int lthread_key_create(unsigned int *key, tls_destructor_func destructor);
/**
* Delete key for lthread TLS
*
* This function is modelled on pthread_key_delete().
* It deletes a thread-specific data key previously returned by
* lthread_key_create().
* The thread-specific data values associated with the key need not be NULL
* at the time that lthread_key_delete is called.
* It is the responsibility of the application to free any application
* storage or perform any cleanup actions for data structures related to the
* deleted key. This cleanup can be done either before or after
* lthread_key_delete is called.
*
* @param key
* The key to be deleted
*
* @return
* 0 Success
* EINVAL the key was invalid
*/
int lthread_key_delete(unsigned int key);
/**
* Get lthread TLS
*
* This function is modelled on pthread_get_specific().
* It returns the value currently bound to the specified key on behalf of the
* calling thread. Calling lthread_getspecific() with a key value not
* obtained from lthread_key_create() or after key has been deleted with
* lthread_key_delete() will result in undefined behaviour.
* lthread_getspecific() may be called from a thread-specific data destructor
* function.
*
* @param key
* The key for which data is requested
*
* @return
* Pointer to the thread specific data associated with that key
* or NULL if no data has been set.
*/
void
*lthread_getspecific(unsigned int key);
/**
* Set lthread TLS
*
* This function is modelled on pthread_set_sepcific()
* It associates a thread-specific value with a key obtained via a previous
* call to lthread_key_create().
* Different threads may bind different values to the same key. These values
* are typically pointers to dynamically allocated memory that have been
* reserved by the calling thread. Calling lthread_setspecific with a key
* value not obtained from lthread_key_create or after the key has been
* deleted with lthread_key_delete will result in undefined behaviour.
*
* @param key
* The key for which data is to be set
* @param key
* Pointer to the user data
*
* @return
* 0 success
* EINVAL the key was invalid
*/
int lthread_setspecific(unsigned int key, const void *value);
/**
* The macros below provide an alternative mechanism to access lthread local
* storage.
*
* The macros can be used to declare define and access per lthread local
* storage in a similar way to the RTE_PER_LCORE macros which control storage
* local to an lcore.
*
* Memory for per lthread variables declared in this way is allocated when the
* lthread is created and a pointer to this memory is stored in the lthread.
* The per lthread variables are accessed via the pointer + the offset of the
* particular variable.
*
* The total size of per lthread storage, and the variable offsets are found by
* defining the variables in a unique global memory section, the start and end
* of which is known. This global memory section is used only in the
* computation of the addresses of the lthread variables, and is never actually
* used to store any data.
*
* Due to the fact that variables declared this way may be scattered across
* many files, the start and end of the section and variable offsets are only
* known after linking, thus the computation of section size and variable
* addresses is performed at run time.
*
* These macros are primarily provided to aid porting of code that makes use
* of the existing RTE_PER_LCORE macros. In principle it would be more efficient
* to gather all lthread local variables into a single structure and
* set/retrieve a pointer to that struct using the alternative
* lthread_data_set/get APIs.
*
* These macros are mutually exclusive with the lthread_data_set/get APIs.
* If you define storage using these macros then the lthread_data_set/get APIs
* will not perform as expected, the lthread_data_set API does nothing, and the
* lthread_data_get API returns the start of global section.
*
*/
/* start and end of per lthread section */
extern char __start_per_lt;
extern char __stop_per_lt;
#define RTE_DEFINE_PER_LTHREAD(type, name) \
__typeof__(type)__attribute((section("per_lt"))) per_lt_##name
/**
* Macro to declare an extern per lthread variable "var" of type "type"
*/
#define RTE_DECLARE_PER_LTHREAD(type, name) \
extern __typeof__(type)__attribute((section("per_lt"))) per_lt_##name
/**
* Read/write the per-lcore variable value
*/
#define RTE_PER_LTHREAD(name) ((typeof(per_lt_##name) *)\
((char *)lthread_get_data() +\
((char *) &per_lt_##name - &__start_per_lt)))
/**
* Initialize a mutex
*
* This function provides a mutual exclusion device, the need for which
* can normally be avoided in a cooperative multitasking environment.
* It is provided to aid porting of legacy code originally written for
* preemptive multitasking environments such as pthreads.
*
* A mutex may be unlocked (not owned by any thread), or locked (owned by
* one thread).
*
* A mutex can never be owned by more than one thread simultaneously.
* A thread attempting to lock a mutex that is already locked by another
* thread is suspended until the owning thread unlocks the mutex.
*
* lthread_mutex_init() initializes the mutex object pointed to by mutex
* Optional mutex attributes specified in mutexattr, are reserved for future
* use and are currently ignored.
*
* If a thread calls lthread_mutex_lock() on the mutex, then if the mutex
* is currently unlocked, it becomes locked and owned by the calling
* thread, and lthread_mutex_lock returns immediately. If the mutex is
* already locked by another thread, lthread_mutex_lock suspends the calling
* thread until the mutex is unlocked.
*
* lthread_mutex_trylock behaves identically to rte_thread_mutex_lock, except
* that it does not block the calling thread if the mutex is already locked
* by another thread.
*
* lthread_mutex_unlock() unlocks the specified mutex. The mutex is assumed
* to be locked and owned by the calling thread.
*
* lthread_mutex_destroy() destroys a mutex object, freeing its resources.
* The mutex must be unlocked with nothing blocked on it before calling
* lthread_mutex_destroy.
*
* @param name
* Optional pointer to string describing the mutex
* @param mutex
* Pointer to pointer to the mutex to be initialized
* @param attribute
* Pointer to attribute - unused reserved
*
* @return
* 0 success
* EINVAL mutex was not a valid pointer
* EAGAIN insufficient resources
*/
int
lthread_mutex_init(char *name, struct lthread_mutex **mutex,
const struct lthread_mutexattr *attr);
/**
* Destroy a mutex
*
* This function destroys the specified mutex freeing its resources.
* The mutex must be unlocked before calling lthread_mutex_destroy.
*
* @see lthread_mutex_init()
*
* @param mutex
* Pointer to pointer to the mutex to be initialized
*
* @return
* 0 success
* EINVAL mutex was not an initialized mutex
* EBUSY mutex was still in use
*/
int lthread_mutex_destroy(struct lthread_mutex *mutex);
/**
* Lock a mutex
*
* This function attempts to lock a mutex.
* If a thread calls lthread_mutex_lock() on the mutex, then if the mutex
* is currently unlocked, it becomes locked and owned by the calling
* thread, and lthread_mutex_lock returns immediately. If the mutex is
* already locked by another thread, lthread_mutex_lock suspends the calling
* thread until the mutex is unlocked.
*
* @see lthread_mutex_init()
*
* @param mutex
* Pointer to pointer to the mutex to be initialized
*
* @return
* 0 success
* EINVAL mutex was not an initialized mutex
* EDEADLOCK the mutex was already owned by the calling thread
*/
int lthread_mutex_lock(struct lthread_mutex *mutex);
/**
* Try to lock a mutex
*
* This function attempts to lock a mutex.
* lthread_mutex_trylock behaves identically to rte_thread_mutex_lock, except
* that it does not block the calling thread if the mutex is already locked
* by another thread.
*
*
* @see lthread_mutex_init()
*
* @param mutex
* Pointer to pointer to the mutex to be initialized
*
* @return
* 0 success
* EINVAL mutex was not an initialized mutex
* EBUSY the mutex was already locked by another thread
*/
int lthread_mutex_trylock(struct lthread_mutex *mutex);
/**
* Unlock a mutex
*
* This function attempts to unlock the specified mutex. The mutex is assumed
* to be locked and owned by the calling thread.
*
* The oldest of any threads blocked on the mutex is made ready and may
* compete with any other running thread to gain the mutex, it fails it will
* be blocked again.
*
* @param mutex
* Pointer to pointer to the mutex to be initialized
*
* @return
* 0 mutex was unlocked
* EINVAL mutex was not an initialized mutex
* EPERM the mutex was not owned by the calling thread
*/
int lthread_mutex_unlock(struct lthread_mutex *mutex);
/**
* Initialize a condition variable
*
* This function initializes a condition variable.
*
* Condition variables can be used to communicate changes in the state of data
* shared between threads.
*
* @see lthread_cond_wait()
*
* @param name
* Pointer to optional string describing the condition variable
* @param c
* Pointer to pointer to the condition variable to be initialized
* @param attr
* Pointer to optional attribute reserved for future use, currently ignored
*
* @return
* 0 success
* EINVAL cond was not a valid pointer
* EAGAIN insufficient resources
*/
int
lthread_cond_init(char *name, struct lthread_cond **c,
const struct lthread_condattr *attr);
/**
* Destroy a condition variable
*
* This function destroys a condition variable that was created with
* lthread_cond_init() and releases its resources.
*
* @param cond
* Pointer to pointer to the condition variable to be destroyed
*
* @return
* 0 Success
* EBUSY condition variable was still in use
* EINVAL was not an initialised condition variable
*/
int lthread_cond_destroy(struct lthread_cond *cond);
/**
* Wait on a condition variable
*
* The function blocks the current thread waiting on the condition variable
* specified by cond. The waiting thread unblocks only after another thread
* calls lthread_cond_signal, or lthread_cond_broadcast, specifying the
* same condition variable.
*
* @param cond
* Pointer to pointer to the condition variable to be waited on
*
* @param reserved
* reserved for future use
*
* @return
* 0 The condition was signalled ( Success )
* EINVAL was not a an initialised condition variable
*/
int lthread_cond_wait(struct lthread_cond *c, uint64_t reserved);
/**
* Signal a condition variable
*
* The function unblocks one thread waiting for the condition variable cond.
* If no threads are waiting on cond, the rte_lthead_cond_signal() function
* has no effect.
*
* @param cond
* Pointer to pointer to the condition variable to be signalled
*
* @return
* 0 The condition was signalled ( Success )
* EINVAL was not a an initialised condition variable
*/
int lthread_cond_signal(struct lthread_cond *c);
/**
* Broadcast a condition variable
*
* The function unblocks all threads waiting for the condition variable cond.
* If no threads are waiting on cond, the rte_lthead_cond_broadcast()
* function has no effect.
*
* @param cond
* Pointer to pointer to the condition variable to be signalled
*
* @return
* 0 The condition was signalled ( Success )
* EINVAL was not a an initialised condition variable
*/
int lthread_cond_broadcast(struct lthread_cond *c);
#endif /* LTHREAD_H */

240
examples/performance-thread/common/lthread_cond.c Normal file
View File

@ -0,0 +1,240 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Some portions of this software may have been derived from the
* https://github.com/halayli/lthread which carrys the following license.
*
* Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <stddef.h>
#include <limits.h>
#include <inttypes.h>
#include <unistd.h>
#include <pthread.h>
#include <fcntl.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <errno.h>
#include <rte_config.h>
#include <rte_log.h>
#include <rte_common.h>
#include "lthread_api.h"
#include "lthread_diag_api.h"
#include "lthread_diag.h"
#include "lthread_int.h"
#include "lthread_sched.h"
#include "lthread_queue.h"
#include "lthread_objcache.h"
#include "lthread_timer.h"
#include "lthread_mutex.h"
#include "lthread_cond.h"
/*
* Create a condition variable
*/
int
lthread_cond_init(char *name, struct lthread_cond **cond,
__rte_unused const struct lthread_condattr *attr)
{
struct lthread_cond *c;
if (cond == NULL)
return POSIX_ERRNO(EINVAL);
/* allocate a condition variable from cache */
c = _lthread_objcache_alloc((THIS_SCHED)->cond_cache);
if (c == NULL)
return POSIX_ERRNO(EAGAIN);
c->blocked = _lthread_queue_create("blocked");
if (c->blocked == NULL) {
_lthread_objcache_free((THIS_SCHED)->cond_cache, (void *)c);
return POSIX_ERRNO(EAGAIN);
}
if (name == NULL)
strncpy(c->name, "no name", sizeof(c->name));
else
strncpy(c->name, name, sizeof(c->name));
c->name[sizeof(c->name)-1] = 0;
c->root_sched = THIS_SCHED;
(*cond) = c;
DIAG_CREATE_EVENT((*cond), LT_DIAG_COND_CREATE);
return 0;
}
/*
* Destroy a condition variable
*/
int lthread_cond_destroy(struct lthread_cond *c)
{
if (c == NULL) {
DIAG_EVENT(c, LT_DIAG_COND_DESTROY, c, POSIX_ERRNO(EINVAL));
return POSIX_ERRNO(EINVAL);
}
/* try to free it */
if (_lthread_queue_destroy(c->blocked) < 0) {
/* queue in use */
DIAG_EVENT(c, LT_DIAG_COND_DESTROY, c, POSIX_ERRNO(EBUSY));
return POSIX_ERRNO(EBUSY);
}
/* okay free it */
_lthread_objcache_free(c->root_sched->cond_cache, c);
DIAG_EVENT(c, LT_DIAG_COND_DESTROY, c, 0);
return 0;
}
/*
* Wait on a condition variable
*/
int lthread_cond_wait(struct lthread_cond *c, __rte_unused uint64_t reserved)
{
struct lthread *lt = THIS_LTHREAD;
if (c == NULL) {
DIAG_EVENT(c, LT_DIAG_COND_WAIT, c, POSIX_ERRNO(EINVAL));
return POSIX_ERRNO(EINVAL);
}
DIAG_EVENT(c, LT_DIAG_COND_WAIT, c, 0);
/* queue the current thread in the blocked queue
* this will be written when we return to the scheduler
* to ensure that the current thread context is saved
* before any signal could result in it being dequeued and
* resumed
*/
lt->pending_wr_queue = c->blocked;
_suspend();
/* the condition happened */
return 0;
}
/*
* Signal a condition variable
* attempt to resume any blocked thread
*/
int lthread_cond_signal(struct lthread_cond *c)
{
struct lthread *lt;
if (c == NULL) {
DIAG_EVENT(c, LT_DIAG_COND_SIGNAL, c, POSIX_ERRNO(EINVAL));
return POSIX_ERRNO(EINVAL);
}
lt = _lthread_queue_remove(c->blocked);
if (lt != NULL) {
/* okay wake up this thread */
DIAG_EVENT(c, LT_DIAG_COND_SIGNAL, c, lt);
_ready_queue_insert((struct lthread_sched *)lt->sched, lt);
}
return 0;
}
/*
* Broadcast a condition variable
*/
int lthread_cond_broadcast(struct lthread_cond *c)
{
struct lthread *lt;
if (c == NULL) {
DIAG_EVENT(c, LT_DIAG_COND_BROADCAST, c, POSIX_ERRNO(EINVAL));
return POSIX_ERRNO(EINVAL);
}
DIAG_EVENT(c, LT_DIAG_COND_BROADCAST, c, 0);
do {
/* drain the queue waking everybody */
lt = _lthread_queue_remove(c->blocked);
if (lt != NULL) {
DIAG_EVENT(c, LT_DIAG_COND_BROADCAST, c, lt);
/* wake up */
_ready_queue_insert((struct lthread_sched *)lt->sched,
lt);
}
} while (!_lthread_queue_empty(c->blocked));
_reschedule();
DIAG_EVENT(c, LT_DIAG_COND_BROADCAST, c, 0);
return 0;
}
/*
* return the diagnostic ref val stored in a condition var
*/
uint64_t
lthread_cond_diag_ref(struct lthread_cond *c)
{
if (c == NULL)
return 0;
return c->diag_ref;
}

77
examples/performance-thread/common/lthread_cond.h Normal file
View File

@ -0,0 +1,77 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Some portions of this software may have been derived from the
* https://github.com/halayli/lthread which carrys the following license.
*
* Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef LTHREAD_COND_H_
#define LTHREAD_COND_H_
#include "lthread_queue.h"
#define MAX_COND_NAME_SIZE 64
struct lthread_cond {
struct lthread_queue *blocked;
struct lthread_sched *root_sched;
int count;
char name[MAX_COND_NAME_SIZE];
uint64_t diag_ref; /* optional ref to user diag data */
} __rte_cache_aligned;
#endif /* LTHREAD_COND_H_ */

324
examples/performance-thread/common/lthread_diag.c Normal file
View File

@ -0,0 +1,324 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <rte_config.h>
#include <rte_log.h>
#include <rte_common.h>
#include "lthread_diag.h"
#include "lthread_queue.h"
#include "lthread_pool.h"
#include "lthread_objcache.h"
#include "lthread_sched.h"
#include "lthread_diag_api.h"
/* dummy ref value of default diagnostic callback */
static uint64_t dummy_ref;
#define DIAG_SCHED_STATS_FORMAT \
"core %d\n%33s %12s %12s %12s %12s\n"
#define DIAG_CACHE_STATS_FORMAT \
"%20s %12lu %12lu %12lu %12lu %12lu\n"
#define DIAG_QUEUE_STATS_FORMAT \
"%20s %12lu %12lu %12lu\n"
/*
* texts used in diagnostic events,
* corresponding diagnostic mask bit positions are given as comment
*/
const char *diag_event_text[] = {
"LTHREAD_CREATE ", /* 00 */
"LTHREAD_EXIT ", /* 01 */
"LTHREAD_JOIN ", /* 02 */
"LTHREAD_CANCEL ", /* 03 */
"LTHREAD_DETACH ", /* 04 */
"LTHREAD_FREE ", /* 05 */
"LTHREAD_SUSPENDED ", /* 06 */
"LTHREAD_YIELD ", /* 07 */
"LTHREAD_RESCHEDULED", /* 08 */
"LTHREAD_SLEEP ", /* 09 */
"LTHREAD_RESUMED ", /* 10 */
"LTHREAD_AFFINITY ", /* 11 */
"LTHREAD_TMR_START ", /* 12 */
"LTHREAD_TMR_DELETE ", /* 13 */
"LTHREAD_TMR_EXPIRED", /* 14 */
"COND_CREATE ", /* 15 */
"COND_DESTROY ", /* 16 */
"COND_WAIT ", /* 17 */
"COND_SIGNAL ", /* 18 */
"COND_BROADCAST ", /* 19 */
"MUTEX_CREATE ", /* 20 */
"MUTEX_DESTROY ", /* 21 */
"MUTEX_LOCK ", /* 22 */
"MUTEX_TRYLOCK ", /* 23 */
"MUTEX_BLOCKED ", /* 24 */
"MUTEX_UNLOCKED ", /* 25 */
"SCHED_CREATE ", /* 26 */
"SCHED_SHUTDOWN " /* 27 */
};
/*
* set diagnostic ,ask
*/
void lthread_diagnostic_set_mask(DIAG_USED uint64_t mask)
{
#if LTHREAD_DIAG
diag_mask = mask;
#else
RTE_LOG(INFO, LTHREAD,
"LTHREAD_DIAG is not set, see lthread_diag_api.h\n");
#endif
}
/*
* Check consistency of the scheduler stats
* Only sensible run after the schedulers are stopped
* Count the number of objects lying in caches and queues
* and available in the qnode pool.
* This should be equal to the total capacity of all
* qnode pools.
*/
void
_sched_stats_consistency_check(void);
void
_sched_stats_consistency_check(void)
{
#if LTHREAD_DIAG
int i;
struct lthread_sched *sched;
uint64_t count = 0;
uint64_t capacity = 0;
for (i = 0; i < LTHREAD_MAX_LCORES; i++) {
sched = schedcore[i];
if (sched == NULL)
continue;
/* each of these queues consumes a stub node */
count += 8;
count += DIAG_COUNT(sched->ready, size);
count += DIAG_COUNT(sched->pready, size);
count += DIAG_COUNT(sched->lthread_cache, available);
count += DIAG_COUNT(sched->stack_cache, available);
count += DIAG_COUNT(sched->tls_cache, available);
count += DIAG_COUNT(sched->per_lthread_cache, available);
count += DIAG_COUNT(sched->cond_cache, available);
count += DIAG_COUNT(sched->mutex_cache, available);
/* the node pool does not consume a stub node */
if (sched->qnode_pool->fast_alloc != NULL)
count++;
count += DIAG_COUNT(sched->qnode_pool, available);
capacity += DIAG_COUNT(sched->qnode_pool, capacity);
}
if (count != capacity) {
RTE_LOG(CRIT, LTHREAD,
"Scheduler caches are inconsistent\n");
} else {
RTE_LOG(INFO, LTHREAD,
"Scheduler caches are ok\n");
}
#endif
}
#if LTHREAD_DIAG
/*
* Display node pool stats
*/
static inline void
_qnode_pool_display(DIAG_USED struct qnode_pool *p)
{
printf(DIAG_CACHE_STATS_FORMAT,
p->name,
DIAG_COUNT(p, rd),
DIAG_COUNT(p, wr),
DIAG_COUNT(p, available),
DIAG_COUNT(p, prealloc),
DIAG_COUNT(p, capacity));
fflush(stdout);
}
#endif
#if LTHREAD_DIAG
/*
* Display queue stats
*/
static inline void
_lthread_queue_display(DIAG_USED struct lthread_queue *q)
{
#if DISPLAY_OBJCACHE_QUEUES
printf(DIAG_QUEUE_STATS_FORMAT,
q->name,
DIAG_COUNT(q, rd),
DIAG_COUNT(q, wr),
DIAG_COUNT(q, size));
fflush(stdout);
#else
printf("%s: queue stats disabled\n",
q->name);
#endif
}
#endif
#if LTHREAD_DIAG
/*
* Display objcache stats
*/
static inline void
_objcache_display(DIAG_USED struct lthread_objcache *c)
{
printf(DIAG_CACHE_STATS_FORMAT,
c->name,
DIAG_COUNT(c, rd),
DIAG_COUNT(c, wr),
DIAG_COUNT(c, available),
DIAG_COUNT(c, prealloc),
DIAG_COUNT(c, capacity));
_lthread_queue_display(c->q);
fflush(stdout);
}
#endif
/*
* Display sched stats
*/
void
lthread_sched_stats_display(void)
{
#if LTHREAD_DIAG
int i;
struct lthread_sched *sched;
for (i = 0; i < LTHREAD_MAX_LCORES; i++) {
sched = schedcore[i];
if (sched != NULL) {
printf(DIAG_SCHED_STATS_FORMAT,
sched->lcore_id,
"rd",
"wr",
"present",
"nb preallocs",
"capacity");
_lthread_queue_display(sched->ready);
_lthread_queue_display(sched->pready);
_qnode_pool_display(sched->qnode_pool);
_objcache_display(sched->lthread_cache);
_objcache_display(sched->stack_cache);
_objcache_display(sched->tls_cache);
_objcache_display(sched->per_lthread_cache);
_objcache_display(sched->cond_cache);
_objcache_display(sched->mutex_cache);
fflush(stdout);
}
}
_sched_stats_consistency_check();
#else
RTE_LOG(INFO, LTHREAD,
"lthread diagnostics disabled\n"
"hint - set LTHREAD_DIAG in lthread_diag_api.h\n");
#endif
}
/*
* Defafult diagnostic callback
*/
static uint64_t
_lthread_diag_default_cb(uint64_t time, struct lthread *lt, int diag_event,
uint64_t diag_ref, const char *text, uint64_t p1, uint64_t p2)
{
uint64_t _p2;
int lcore = (int) rte_lcore_id();
switch (diag_event) {
case LT_DIAG_LTHREAD_CREATE:
case LT_DIAG_MUTEX_CREATE:
case LT_DIAG_COND_CREATE:
_p2 = dummy_ref;
break;
default:
_p2 = p2;
break;
}
printf("%"PRIu64" %d %8.8lx %8.8lx %s %8.8lx %8.8lx\n",
time,
lcore,
(uint64_t) lt,
diag_ref,
text,
p1,
_p2);
return dummy_ref++;
}
/*
* plug in default diag callback with mask off
*/
void _lthread_diag_ctor(void)__attribute__((constructor));
void _lthread_diag_ctor(void)
{
diag_cb = _lthread_diag_default_cb;
diag_mask = 0;
}
/*
* enable diagnostics
*/
void lthread_diagnostic_enable(DIAG_USED diag_callback cb,
DIAG_USED uint64_t mask)
{
#if LTHREAD_DIAG
if (cb == NULL)
diag_cb = _lthread_diag_default_cb;
else
diag_cb = cb;
diag_mask = mask;
#else
RTE_LOG(INFO, LTHREAD,
"LTHREAD_DIAG is not set, see lthread_diag_api.h\n");
#endif
}

132
examples/performance-thread/common/lthread_diag.h Normal file
View File

@ -0,0 +1,132 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef LTHREAD_DIAG_H_
#define LTHREAD_DIAG_H_
#include <stdint.h>
#include <inttypes.h>
#include <rte_log.h>
#include <rte_common.h>
#include "lthread_api.h"
#include "lthread_diag_api.h"
extern diag_callback diag_cb;
extern const char *diag_event_text[];
extern uint64_t diag_mask;
/* max size of name strings */
#define LT_MAX_NAME_SIZE 64
#if LTHREAD_DIAG
#define DISPLAY_OBJCACHE_QUEUES 1
/*
* Generate a diagnostic trace or event in the case where an object is created.
*
* The value returned by the callback is stored in the object.
*
* @ param obj
* pointer to the object that was created
* @ param ev
* the event code
*
*/
#define DIAG_CREATE_EVENT(obj, ev) do { \
struct lthread *ct = RTE_PER_LCORE(this_sched)->current_lthread;\
if ((BIT(ev) & diag_mask) && (ev < LT_DIAG_EVENT_MAX)) { \
(obj)->diag_ref = (diag_cb)(rte_rdtsc(), \
ct, \
(ev), \
0, \
diag_event_text[(ev)], \
(uint64_t)obj, \
0); \
} \
} while (0)
/*
* Generate a diagnostic trace event.
*
* @ param obj
* pointer to the lthread, cond or mutex object
* @ param ev
* the event code
* @ param p1
* object specific value ( see lthread_diag_api.h )
* @ param p2
* object specific value ( see lthread_diag_api.h )
*/
#define DIAG_EVENT(obj, ev, p1, p2) do { \
struct lthread *ct = RTE_PER_LCORE(this_sched)->current_lthread;\
if ((BIT(ev) & diag_mask) && (ev < LT_DIAG_EVENT_MAX)) { \
(diag_cb)(rte_rdtsc(), \
ct, \
ev, \
(obj)->diag_ref, \
diag_event_text[(ev)], \
(uint64_t)(p1), \
(uint64_t)(p2)); \
} \
} while (0)
#define DIAG_COUNT_DEFINE(x) rte_atomic64_t count_##x
#define DIAG_COUNT_INIT(o, x) rte_atomic64_init(&((o)->count_##x))
#define DIAG_COUNT_INC(o, x) rte_atomic64_inc(&((o)->count_##x))
#define DIAG_COUNT_DEC(o, x) rte_atomic64_dec(&((o)->count_##x))
#define DIAG_COUNT(o, x) rte_atomic64_read(&((o)->count_##x))
#define DIAG_USED
#else
/* no diagnostics configured */
#define DISPLAY_OBJCACHE_QUEUES 0
#define DIAG_CREATE_EVENT(obj, ev)
#define DIAG_EVENT(obj, ev, p1, p)
#define DIAG_COUNT_DEFINE(x)
#define DIAG_COUNT_INIT(o, x) do {} while (0)
#define DIAG_COUNT_INC(o, x) do {} while (0)
#define DIAG_COUNT_DEC(o, x) do {} while (0)
#define DIAG_COUNT(o, x) 0
#define DIAG_USED __rte_unused
#endif /* LTHREAD_DIAG */
#endif /* LTHREAD_DIAG_H_ */

319
examples/performance-thread/common/lthread_diag_api.h Normal file
View File

@ -0,0 +1,319 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef LTHREAD_DIAG_API_H_
#define LTHREAD_DIAG_API_H_
#include <stdint.h>
#include <inttypes.h>
/*
* Enable diagnostics
* 0 = conditionally compiled out
* 1 = compiled in and maskable at run time, see below for details
*/
#define LTHREAD_DIAG 0
/**
* lthread diagnostic interface
*
* If enabled via configuration file option ( tbd ) the lthread subsystem
* can generate selected trace information, either RTE_LOG (INFO) messages,
* or else invoke a user supplied callback function when any of the events
* listed below occur.
*
* Reporting of events can be selectively masked, the bit position in the
* mask is determined by the corresponding event identifier listed below.
*
* Diagnostics are enabled by registering the callback function and mask
* using the API lthread_diagnostic_enable().
*
* Various interesting parameters are passed to the callback, including the
* time in cpu clks, the lthread id, the diagnostic event id, a user ref value,
* event text string, object being traced, and two context dependent parameters
* (p1 and p2). The meaning of the two parameters p1 and p2 depends on
* the specific event.
*
* The events LT_DIAG_LTHREAD_CREATE, LT_DIAG_MUTEX_CREATE and
* LT_DIAG_COND_CREATE are implicitly enabled if the event mask includes any of
* the LT_DIAG_LTHREAD_XXX, LT_DIAG_MUTEX_XXX or LT_DIAG_COND_XXX events
* respectively.
*
* These create events may also be included in the mask discreetly if it is
* desired to monitor only create events.
*
* @param time
* The time in cpu clks at which the event occurred
*
* @param lthread
* The current lthread
*
* @param diag_event
* The diagnostic event id (bit position in the mask)
*
* @param diag_ref
*
* For LT_DIAG_LTHREAD_CREATE, LT_DIAG_MUTEX_CREATE or LT_DIAG_COND_CREATE
* this parameter is not used and set to 0.
* All other events diag_ref contains the user ref value returned by the
* callback function when lthread is created.
*
* The diag_ref values assigned to mutex and cond var can be retrieved
* using the APIs lthread_mutex_diag_ref(), and lthread_cond_diag_ref()
* respectively.
*
* @param p1
* see below
*
* @param p1
* see below
*
* @returns
* For LT_DIAG_LTHREAD_CREATE, LT_DIAG_MUTEX_CREATE or LT_DIAG_COND_CREATE
* expects a user diagnostic ref value that will be saved in the lthread, mutex
* or cond var.
*
* For all other events return value is ignored.
*
* LT_DIAG_SCHED_CREATE - Invoked when a scheduler is created
* p1 = the scheduler that was created
* p2 = not used
* return value will be ignored
*
* LT_DIAG_SCHED_SHUTDOWN - Invoked when a shutdown request is received
* p1 = the scheduler to be shutdown
* p2 = not used
* return value will be ignored
*
* LT_DIAG_LTHREAD_CREATE - Invoked when a thread is created
* p1 = the lthread that was created
* p2 = not used
* return value will be stored in the lthread
*
* LT_DIAG_LTHREAD_EXIT - Invoked when a lthread exits
* p2 = 0 if the thread was already joined
* p2 = 1 if the thread was not already joined
* return val ignored
*
* LT_DIAG_LTHREAD_JOIN - Invoked when a lthread exits
* p1 = the lthread that is being joined
* p2 = 0 if the thread was already exited
* p2 = 1 if the thread was not already exited
* return val ignored
*
* LT_DIAG_LTHREAD_CANCELLED - Invoked when an lthread is cancelled
* p1 = not used
* p2 = not used
* return val ignored
*
* LT_DIAG_LTHREAD_DETACH - Invoked when an lthread is detached
* p1 = not used
* p2 = not used
* return val ignored
*
* LT_DIAG_LTHREAD_FREE - Invoked when an lthread is freed
* p1 = not used
* p2 = not used
* return val ignored
*
* LT_DIAG_LTHREAD_SUSPENDED - Invoked when an lthread is suspended
* p1 = not used
* p2 = not used
* return val ignored
*
* LT_DIAG_LTHREAD_YIELD - Invoked when an lthread explicitly yields
* p1 = not used
* p2 = not used
* return val ignored
*
* LT_DIAG_LTHREAD_RESCHEDULED - Invoked when an lthread is rescheduled
* p1 = not used
* p2 = not used
* return val ignored
*
* LT_DIAG_LTHREAD_RESUMED - Invoked when an lthread is resumed
* p1 = not used
* p2 = not used
* return val ignored
*
* LT_DIAG_LTHREAD_AFFINITY - Invoked when an lthread is affinitised
* p1 = the destination lcore_id
* p2 = not used
* return val ignored
*
* LT_DIAG_LTHREAD_TMR_START - Invoked when an lthread starts a timer
* p1 = address of timer node
* p2 = the timeout value
* return val ignored
*
* LT_DIAG_LTHREAD_TMR_DELETE - Invoked when an lthread deletes a timer
* p1 = address of the timer node
* p2 = 0 the timer and the was successfully deleted
* p2 = not usee
* return val ignored
*
* LT_DIAG_LTHREAD_TMR_EXPIRED - Invoked when an lthread timer expires
* p1 = address of scheduler the timer expired on
* p2 = the thread associated with the timer
* return val ignored
*
* LT_DIAG_COND_CREATE - Invoked when a condition variable is created
* p1 = address of cond var that was created
* p2 = not used
* return diag ref value will be stored in the condition variable
*
* LT_DIAG_COND_DESTROY - Invoked when a condition variable is destroyed
* p1 = not used
* p2 = not used
* return val ignored
*
* LT_DIAG_COND_WAIT - Invoked when an lthread waits on a cond var
* p1 = the address of the condition variable
* p2 = not used
* return val ignored
*
* LT_DIAG_COND_SIGNAL - Invoked when an lthread signals a cond var
* p1 = the address of the cond var
* p2 = the lthread that was signalled, or error code
* return val ignored
*
* LT_DIAG_COND_BROADCAST - Invoked when an lthread broadcasts a cond var
* p1 = the address of the condition variable
* p2 = the lthread(s) that are signalled, or error code
*
* LT_DIAG_MUTEX_CREATE - Invoked when a mutex is created
* p1 = address of muex
* p2 = not used
* return diag ref value will be stored in the mutex variable
*
* LT_DIAG_MUTEX_DESTROY - Invoked when a mutex is destroyed
* p1 = address of mutex
* p2 = not used
* return val ignored
*
* LT_DIAG_MUTEX_LOCK - Invoked when a mutex lock is obtained
* p1 = address of mutex
* p2 = function return value
* return val ignored
*
* LT_DIAG_MUTEX_BLOCKED - Invoked when an lthread blocks on a mutex
* p1 = address of mutex
* p2 = function return value
* return val ignored
*
* LT_DIAG_MUTEX_TRYLOCK - Invoked when a mutex try lock is attempted
* p1 = address of mutex
* p2 = the function return value
* return val ignored
*
* LT_DIAG_MUTEX_UNLOCKED - Invoked when a mutex is unlocked
* p1 = address of mutex
* p2 = the thread that was unlocked, or error code
* return val ignored
*/
typedef uint64_t (*diag_callback) (uint64_t time, struct lthread *lt,
int diag_event, uint64_t diag_ref,
const char *text, uint64_t p1, uint64_t p2);
/*
* Set user diagnostic callback and mask
* If the callback function pointer is NULL the default
* callback handler will be restored.
*/
void lthread_diagnostic_enable(diag_callback cb, uint64_t diag_mask);
/*
* Set diagnostic mask
*/
void lthread_diagnostic_set_mask(uint64_t mask);
/*
* lthread diagnostic callback
*/
enum lthread_diag_ev {
/* bits 0 - 14 lthread flag group */
LT_DIAG_LTHREAD_CREATE, /* 00 mask 0x00000001 */
LT_DIAG_LTHREAD_EXIT, /* 01 mask 0x00000002 */
LT_DIAG_LTHREAD_JOIN, /* 02 mask 0x00000004 */
LT_DIAG_LTHREAD_CANCEL, /* 03 mask 0x00000008 */
LT_DIAG_LTHREAD_DETACH, /* 04 mask 0x00000010 */
LT_DIAG_LTHREAD_FREE, /* 05 mask 0x00000020 */
LT_DIAG_LTHREAD_SUSPENDED, /* 06 mask 0x00000040 */
LT_DIAG_LTHREAD_YIELD, /* 07 mask 0x00000080 */
LT_DIAG_LTHREAD_RESCHEDULED, /* 08 mask 0x00000100 */
LT_DIAG_LTHREAD_SLEEP, /* 09 mask 0x00000200 */
LT_DIAG_LTHREAD_RESUMED, /* 10 mask 0x00000400 */
LT_DIAG_LTHREAD_AFFINITY, /* 11 mask 0x00000800 */
LT_DIAG_LTHREAD_TMR_START, /* 12 mask 0x00001000 */
LT_DIAG_LTHREAD_TMR_DELETE, /* 13 mask 0x00002000 */
LT_DIAG_LTHREAD_TMR_EXPIRED, /* 14 mask 0x00004000 */
/* bits 15 - 19 conditional variable flag group */
LT_DIAG_COND_CREATE, /* 15 mask 0x00008000 */
LT_DIAG_COND_DESTROY, /* 16 mask 0x00010000 */
LT_DIAG_COND_WAIT, /* 17 mask 0x00020000 */
LT_DIAG_COND_SIGNAL, /* 18 mask 0x00040000 */
LT_DIAG_COND_BROADCAST, /* 19 mask 0x00080000 */
/* bits 20 - 25 mutex flag group */
LT_DIAG_MUTEX_CREATE, /* 20 mask 0x00100000 */
LT_DIAG_MUTEX_DESTROY, /* 21 mask 0x00200000 */
LT_DIAG_MUTEX_LOCK, /* 22 mask 0x00400000 */
LT_DIAG_MUTEX_TRYLOCK, /* 23 mask 0x00800000 */
LT_DIAG_MUTEX_BLOCKED, /* 24 mask 0x01000000 */
LT_DIAG_MUTEX_UNLOCKED, /* 25 mask 0x02000000 */
/* bits 26 - 27 scheduler flag group - 8 bits */
LT_DIAG_SCHED_CREATE, /* 26 mask 0x04000000 */
LT_DIAG_SCHED_SHUTDOWN, /* 27 mask 0x08000000 */
LT_DIAG_EVENT_MAX
};
#define LT_DIAG_ALL 0xffffffffffffffff
/*
* Display scheduler stats
*/
void
lthread_sched_stats_display(void);
/*
* return the diagnostic ref val stored in a condition var
*/
uint64_t
lthread_cond_diag_ref(struct lthread_cond *c);
/*
* return the diagnostic ref val stored in a mutex
*/
uint64_t
lthread_mutex_diag_ref(struct lthread_mutex *m);
#endif /* LTHREAD_DIAG_API_H_ */

212
examples/performance-thread/common/lthread_int.h Normal file
View File

@ -0,0 +1,212 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Some portions of this software may have been derived from the
* https://github.com/halayli/lthread which carrys the following license.
*
* Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef LTHREAD_INT_H
#include <lthread_api.h>
#define LTHREAD_INT_H
#include <stdint.h>
#include <sys/time.h>
#include <sys/types.h>
#include <errno.h>
#include <pthread.h>
#include <time.h>
#include <rte_cycles.h>
#include <rte_per_lcore.h>
#include <rte_timer.h>
#include <rte_ring.h>
#include <rte_atomic_64.h>
#include <rte_spinlock.h>
#include <ctx.h>
#include <lthread_api.h>
#include "lthread.h"
#include "lthread_diag.h"
#include "lthread_tls.h"
struct lthread;
struct lthread_sched;
struct lthread_cond;
struct lthread_mutex;
struct lthread_key;
struct key_pool;
struct qnode;
struct qnode_pool;
struct lthread_sched;
struct lthread_tls;
#define BIT(x) (1 << (x))
#define CLEARBIT(x) ~(1 << (x))
#define POSIX_ERRNO(x) (x)
#define MAX_LTHREAD_NAME_SIZE 64
#define RTE_LOGTYPE_LTHREAD RTE_LOGTYPE_USER1
/* define some shorthand for current scheduler and current thread */
#define THIS_SCHED RTE_PER_LCORE(this_sched)
#define THIS_LTHREAD RTE_PER_LCORE(this_sched)->current_lthread
/*
* Definition of an scheduler struct
*/
struct lthread_sched {
struct ctx ctx; /* cpu context */
uint64_t birth; /* time created */
struct lthread *current_lthread; /* running thread */
unsigned lcore_id; /* this sched lcore */
int run_flag; /* sched shutdown */
uint64_t nb_blocked_threads; /* blocked threads */
struct lthread_queue *ready; /* local ready queue */
struct lthread_queue *pready; /* peer ready queue */
struct lthread_objcache *lthread_cache; /* free lthreads */
struct lthread_objcache *stack_cache; /* free stacks */
struct lthread_objcache *per_lthread_cache; /* free per lthread */
struct lthread_objcache *tls_cache; /* free TLS */
struct lthread_objcache *cond_cache; /* free cond vars */
struct lthread_objcache *mutex_cache; /* free mutexes */
struct qnode_pool *qnode_pool; /* pool of queue nodes */
struct key_pool *key_pool; /* pool of free TLS keys */
size_t stack_size;
uint64_t diag_ref; /* diag ref */
} __rte_cache_aligned;
RTE_DECLARE_PER_LCORE(struct lthread_sched *, this_sched);
/*
* State for an lthread
*/
enum lthread_st {
ST_LT_INIT, /* initial state */
ST_LT_READY, /* lthread is ready to run */
ST_LT_SLEEPING, /* lthread is sleeping */
ST_LT_EXPIRED, /* lthread timeout has expired */
ST_LT_EXITED, /* lthread has exited and needs cleanup */
ST_LT_DETACH, /* lthread frees on exit*/
ST_LT_CANCELLED, /* lthread has been cancelled */
};
/*
* lthread sub states for exit/join
*/
enum join_st {
LT_JOIN_INITIAL, /* initial state */
LT_JOIN_EXITING, /* thread is exiting */
LT_JOIN_THREAD_SET, /* joining thread has been set */
LT_JOIN_EXIT_VAL_SET, /* exiting thread has set ret val */
LT_JOIN_EXIT_VAL_READ, /* joining thread has collected ret val */
};
/* defnition of an lthread stack object */
struct lthread_stack {
uint8_t stack[LTHREAD_MAX_STACK_SIZE];
size_t stack_size;
struct lthread_sched *root_sched;
} __rte_cache_aligned;
/*
* Definition of an lthread
*/
struct lthread {
struct ctx ctx; /* cpu context */
uint64_t state; /* current lthread state */
struct lthread_sched *sched; /* current scheduler */
void *stack; /* ptr to actual stack */
size_t stack_size; /* current stack_size */
size_t last_stack_size; /* last yield stack_size */
lthread_func_t fun; /* func ctx is running */
void *arg; /* func args passed to func */
void *per_lthread_data; /* per lthread user data */
lthread_exit_func exit_handler; /* called when thread exits */
uint64_t birth; /* time lthread was born */
struct lthread_queue *pending_wr_queue; /* deferred queue to write */
struct lthread *lt_join; /* lthread to join on */
uint64_t join; /* state for joining */
void **lt_exit_ptr; /* exit ptr for lthread_join */
struct lthread_sched *root_sched; /* thread was created here*/
struct queue_node *qnode; /* node when in a queue */
struct rte_timer tim; /* sleep timer */
struct lthread_tls *tls; /* keys in use by the thread */
struct lthread_stack *stack_container; /* stack */
char funcname[MAX_LTHREAD_NAME_SIZE]; /* thread func name */
uint64_t diag_ref; /* ref to user diag data */
} __rte_cache_aligned;
/*
* Assert
*/
#if LTHREAD_DIAG
#define LTHREAD_ASSERT(expr) do { \
if (!(expr)) \
rte_panic("line%d\tassert \"" #expr "\" failed\n", __LINE__);\
} while (0)
#else
#define LTHREAD_ASSERT(expr) do {} while (0)
#endif
#endif /* LTHREAD_INT_H */

255
examples/performance-thread/common/lthread_mutex.c Normal file
View File

@ -0,0 +1,255 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <stddef.h>
#include <limits.h>
#include <inttypes.h>
#include <unistd.h>
#include <pthread.h>
#include <fcntl.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <rte_config.h>
#include <rte_per_lcore.h>
#include <rte_log.h>
#include <rte_spinlock.h>
#include <rte_common.h>
#include "lthread_api.h"
#include "lthread_int.h"
#include "lthread_mutex.h"
#include "lthread_sched.h"
#include "lthread_queue.h"
#include "lthread_objcache.h"
#include "lthread_diag.h"
/*
* Create a mutex
*/
int
lthread_mutex_init(char *name, struct lthread_mutex **mutex,
__rte_unused const struct lthread_mutexattr *attr)
{
struct lthread_mutex *m;
if (mutex == NULL)
return POSIX_ERRNO(EINVAL);
m = _lthread_objcache_alloc((THIS_SCHED)->mutex_cache);
if (m == NULL)
return POSIX_ERRNO(EAGAIN);
m->blocked = _lthread_queue_create("blocked queue");
if (m->blocked == NULL) {
_lthread_objcache_free((THIS_SCHED)->mutex_cache, m);
return POSIX_ERRNO(EAGAIN);
}
if (name == NULL)
strncpy(m->name, "no name", sizeof(m->name));
else
strncpy(m->name, name, sizeof(m->name));
m->name[sizeof(m->name)-1] = 0;
m->root_sched = THIS_SCHED;
m->owner = NULL;
rte_atomic64_init(&m->count);
DIAG_CREATE_EVENT(m, LT_DIAG_MUTEX_CREATE);
/* success */
(*mutex) = m;
return 0;
}
/*
* Destroy a mutex
*/
int lthread_mutex_destroy(struct lthread_mutex *m)
{
if ((m == NULL) || (m->blocked == NULL)) {
DIAG_EVENT(m, LT_DIAG_MUTEX_DESTROY, m, POSIX_ERRNO(EINVAL));
return POSIX_ERRNO(EINVAL);
}
if (m->owner == NULL) {
/* try to delete the blocked queue */
if (_lthread_queue_destroy(m->blocked) < 0) {
DIAG_EVENT(m, LT_DIAG_MUTEX_DESTROY,
m, POSIX_ERRNO(EBUSY));
return POSIX_ERRNO(EBUSY);
}
/* free the mutex to cache */
_lthread_objcache_free(m->root_sched->mutex_cache, m);
DIAG_EVENT(m, LT_DIAG_MUTEX_DESTROY, m, 0);
return 0;
}
/* can't do its still in use */
DIAG_EVENT(m, LT_DIAG_MUTEX_DESTROY, m, POSIX_ERRNO(EBUSY));
return POSIX_ERRNO(EBUSY);
}
/*
* Try to obtain a mutex
*/
int lthread_mutex_lock(struct lthread_mutex *m)
{
struct lthread *lt = THIS_LTHREAD;
if ((m == NULL) || (m->blocked == NULL)) {
DIAG_EVENT(m, LT_DIAG_MUTEX_LOCK, m, POSIX_ERRNO(EINVAL));
return POSIX_ERRNO(EINVAL);
}
/* allow no recursion */
if (m->owner == lt) {
DIAG_EVENT(m, LT_DIAG_MUTEX_LOCK, m, POSIX_ERRNO(EDEADLK));
return POSIX_ERRNO(EDEADLK);
}
for (;;) {
rte_atomic64_inc(&m->count);
do {
if (rte_atomic64_cmpset
((uint64_t *) &m->owner, 0, (uint64_t) lt)) {
/* happy days, we got the lock */
DIAG_EVENT(m, LT_DIAG_MUTEX_LOCK, m, 0);
return 0;
}
/* spin due to race with unlock when
* nothing was blocked
*/
} while ((rte_atomic64_read(&m->count) == 1) &&
(m->owner == NULL));
/* queue the current thread in the blocked queue
* we defer this to after we return to the scheduler
* to ensure that the current thread context is saved
* before unlock could result in it being dequeued and
* resumed
*/
DIAG_EVENT(m, LT_DIAG_MUTEX_BLOCKED, m, lt);
lt->pending_wr_queue = m->blocked;
/* now relinquish cpu */
_suspend();
/* resumed, must loop and compete for the lock again */
}
LTHREAD_ASSERT(0);
return 0;
}
/* try to lock a mutex but dont block */
int lthread_mutex_trylock(struct lthread_mutex *m)
{
struct lthread *lt = THIS_LTHREAD;
if ((m == NULL) || (m->blocked == NULL)) {
DIAG_EVENT(m, LT_DIAG_MUTEX_TRYLOCK, m, POSIX_ERRNO(EINVAL));
return POSIX_ERRNO(EINVAL);
}
if (m->owner == lt) {
/* no recursion */
DIAG_EVENT(m, LT_DIAG_MUTEX_TRYLOCK, m, POSIX_ERRNO(EDEADLK));
return POSIX_ERRNO(EDEADLK);
}
rte_atomic64_inc(&m->count);
if (rte_atomic64_cmpset
((uint64_t *) &m->owner, (uint64_t) NULL, (uint64_t) lt)) {
/* got the lock */
DIAG_EVENT(m, LT_DIAG_MUTEX_TRYLOCK, m, 0);
return 0;
}
/* failed so return busy */
rte_atomic64_dec(&m->count);
DIAG_EVENT(m, LT_DIAG_MUTEX_TRYLOCK, m, POSIX_ERRNO(EBUSY));
return POSIX_ERRNO(EBUSY);
}
/*
* Unlock a mutex
*/
int lthread_mutex_unlock(struct lthread_mutex *m)
{
struct lthread *lt = THIS_LTHREAD;
struct lthread *unblocked;
if ((m == NULL) || (m->blocked == NULL)) {
DIAG_EVENT(m, LT_DIAG_MUTEX_UNLOCKED, m, POSIX_ERRNO(EINVAL));
return POSIX_ERRNO(EINVAL);
}
/* fail if its owned */
if (m->owner != lt || m->owner == NULL) {
DIAG_EVENT(m, LT_DIAG_MUTEX_UNLOCKED, m, POSIX_ERRNO(EPERM));
return POSIX_ERRNO(EPERM);
}
rte_atomic64_dec(&m->count);
/* if there are blocked threads then make one ready */
while (rte_atomic64_read(&m->count) > 0) {
unblocked = _lthread_queue_remove(m->blocked);
if (unblocked != NULL) {
rte_atomic64_dec(&m->count);
DIAG_EVENT(m, LT_DIAG_MUTEX_UNLOCKED, m, unblocked);
LTHREAD_ASSERT(unblocked->sched != NULL);
_ready_queue_insert((struct lthread_sched *)
unblocked->sched, unblocked);
break;
}
}
/* release the lock */
m->owner = NULL;
return 0;
}
/*
* return the diagnostic ref val stored in a mutex
*/
uint64_t
lthread_mutex_diag_ref(struct lthread_mutex *m)
{
if (m == NULL)
return 0;
return m->diag_ref;
}

52
examples/performance-thread/common/lthread_mutex.h Normal file
View File

@ -0,0 +1,52 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef LTHREAD_MUTEX_H_
#define LTHREAD_MUTEX_H_
#include "lthread_queue.h"
#define MAX_MUTEX_NAME_SIZE 64
struct lthread_mutex {
struct lthread *owner;
rte_atomic64_t count;
struct lthread_queue *blocked __rte_cache_aligned;
struct lthread_sched *root_sched;
char name[MAX_MUTEX_NAME_SIZE];
uint64_t diag_ref; /* optional ref to user diag data */
} __rte_cache_aligned;
#endif /* LTHREAD_MUTEX_H_ */

158
examples/performance-thread/common/lthread_objcache.h Normal file
View File

@ -0,0 +1,158 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef LTHREAD_OBJCACHE_H_
#define LTHREAD_OBJCACHE_H_
#include <string.h>
#include <rte_per_lcore.h>
#include <rte_malloc.h>
#include <rte_memory.h>
#include "lthread_int.h"
#include "lthread_diag.h"
#include "lthread_queue.h"
RTE_DECLARE_PER_LCORE(struct lthread_sched *, this_sched);
struct lthread_objcache {
struct lthread_queue *q;
size_t obj_size;
int prealloc_size;
char name[LT_MAX_NAME_SIZE];
DIAG_COUNT_DEFINE(rd);
DIAG_COUNT_DEFINE(wr);
DIAG_COUNT_DEFINE(prealloc);
DIAG_COUNT_DEFINE(capacity);
DIAG_COUNT_DEFINE(available);
};
/*
* Create a cache
*/
static inline struct
lthread_objcache *_lthread_objcache_create(const char *name,
size_t obj_size,
int prealloc_size)
{
struct lthread_objcache *c =
rte_malloc_socket(NULL, sizeof(struct lthread_objcache),
RTE_CACHE_LINE_SIZE,
rte_socket_id());
if (c == NULL)
return NULL;
c->q = _lthread_queue_create("cache queue");
if (c->q == NULL) {
rte_free(c);
return NULL;
}
c->obj_size = obj_size;
c->prealloc_size = prealloc_size;
if (name != NULL)
strncpy(c->name, name, LT_MAX_NAME_SIZE);
c->name[sizeof(c->name)-1] = 0;
DIAG_COUNT_INIT(c, rd);
DIAG_COUNT_INIT(c, wr);
DIAG_COUNT_INIT(c, prealloc);
DIAG_COUNT_INIT(c, capacity);
DIAG_COUNT_INIT(c, available);
return c;
}
/*
* Destroy an objcache
*/
static inline int
_lthread_objcache_destroy(struct lthread_objcache *c)
{
if (_lthread_queue_destroy(c->q) == 0) {
rte_free(c);
return 0;
}
return -1;
}
/*
* Allocate an object from an object cache
*/
static inline void *
_lthread_objcache_alloc(struct lthread_objcache *c)
{
int i;
void *data;
struct lthread_queue *q = c->q;
size_t obj_size = c->obj_size;
int prealloc_size = c->prealloc_size;
data = _lthread_queue_remove(q);
if (data == NULL) {
DIAG_COUNT_INC(c, prealloc);
for (i = 0; i < prealloc_size; i++) {
data =
rte_zmalloc_socket(NULL, obj_size,
RTE_CACHE_LINE_SIZE,
rte_socket_id());
if (data == NULL)
return NULL;
DIAG_COUNT_INC(c, available);
DIAG_COUNT_INC(c, capacity);
_lthread_queue_insert_mp(q, data);
}
data = _lthread_queue_remove(q);
}
DIAG_COUNT_INC(c, rd);
DIAG_COUNT_DEC(c, available);
return data;
}
/*
* free an object to a cache
*/
static inline void
_lthread_objcache_free(struct lthread_objcache *c, void *obj)
{
DIAG_COUNT_INC(c, wr);
DIAG_COUNT_INC(c, available);
_lthread_queue_insert_mp(c->q, obj);
}
#endif /* LTHREAD_OBJCACHE_H_ */

332
examples/performance-thread/common/lthread_pool.h Normal file
View File

@ -0,0 +1,332 @@
/*
*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Some portions of this software is derived from the producer
* consumer queues described by Dmitry Vyukov and published here
* http://www.1024cores.net
*
* Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DMITRY VYUKOV OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
* OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and documentation are
* those of the authors and should not be interpreted as representing official
* policies, either expressed or implied, of Dmitry Vyukov.
*/
#ifndef LTHREAD_POOL_H_
#define LTHREAD_POOL_H_
#include <rte_malloc.h>
#include <rte_per_lcore.h>
#include <rte_log.h>
#include "lthread_int.h"
#include "lthread_diag.h"
/*
* This file implements pool of queue nodes used by the queue implemented
* in lthread_queue.h.
*
* The pool is an intrusive lock free MPSC queue.
*
* The pool is created empty and populated lazily, i.e. on first attempt to
* allocate a the pool.
*
* Whenever the pool is empty more nodes are added to the pool
* The number of nodes preallocated in this way is a parameter of
* _qnode_pool_create. Freeing an object returns it to the pool.
*
* Each lthread scheduler maintains its own pool of nodes. L-threads must always
* allocate from this local pool ( because it is a single consumer queue ).
* L-threads can free nodes to any pool (because it is a multi producer queue)
* This enables threads that have affined to a different scheduler to free
* nodes safely.
*/
struct qnode;
struct qnode_cache;
/*
* define intermediate node
*/
struct qnode {
struct qnode *next;
void *data;
struct qnode_pool *pool;
} __rte_cache_aligned;
/*
* a pool structure
*/
struct qnode_pool {
struct qnode *head;
struct qnode *stub;
struct qnode *fast_alloc;
struct qnode *tail __rte_cache_aligned;
int pre_alloc;
char name[LT_MAX_NAME_SIZE];
DIAG_COUNT_DEFINE(rd);
DIAG_COUNT_DEFINE(wr);
DIAG_COUNT_DEFINE(available);
DIAG_COUNT_DEFINE(prealloc);
DIAG_COUNT_DEFINE(capacity);
} __rte_cache_aligned;
/*
* Create a pool of qnodes
*/
static inline struct qnode_pool *
_qnode_pool_create(const char *name, int prealloc_size) {
struct qnode_pool *p = rte_malloc_socket(NULL,
sizeof(struct qnode_pool),
RTE_CACHE_LINE_SIZE,
rte_socket_id());
LTHREAD_ASSERT(p);
p->stub = rte_malloc_socket(NULL,
sizeof(struct qnode),
RTE_CACHE_LINE_SIZE,
rte_socket_id());
LTHREAD_ASSERT(p->stub);
if (name != NULL)
strncpy(p->name, name, LT_MAX_NAME_SIZE);
p->name[sizeof(p->name)-1] = 0;
p->stub->pool = p;
p->stub->next = NULL;
p->tail = p->stub;
p->head = p->stub;
p->pre_alloc = prealloc_size;
DIAG_COUNT_INIT(p, rd);
DIAG_COUNT_INIT(p, wr);
DIAG_COUNT_INIT(p, available);
DIAG_COUNT_INIT(p, prealloc);
DIAG_COUNT_INIT(p, capacity);
return p;
}
/*
* Insert a node into the pool
*/
static inline void __attribute__ ((always_inline))
_qnode_pool_insert(struct qnode_pool *p, struct qnode *n)
{
n->next = NULL;
struct qnode *prev = n;
/* We insert at the head */
prev = (struct qnode *) __sync_lock_test_and_set((uint64_t *)&p->head,
(uint64_t) prev);
/* there is a window of inconsistency until prev next is set */
/* which is why remove must retry */
prev->next = (n);
}
/*
* Remove a node from the pool
*
* There is a race with _qnode_pool_insert() whereby the queue could appear
* empty during a concurrent insert, this is handled by retrying
*
* The queue uses a stub node, which must be swung as the queue becomes
* empty, this requires an insert of the stub, which means that removing the
* last item from the queue incurs the penalty of an atomic exchange. Since the
* pool is maintained with a bulk pre-allocation the cost of this is amortised.
*/
static inline struct qnode *__attribute__ ((always_inline))
_pool_remove(struct qnode_pool *p)
{
struct qnode *head;
struct qnode *tail = p->tail;
struct qnode *next = tail->next;
/* we remove from the tail */
if (tail == p->stub) {
if (next == NULL)
return NULL;
/* advance the tail */
p->tail = next;
tail = next;
next = next->next;
}
if (likely(next != NULL)) {
p->tail = next;
return tail;
}
head = p->head;
if (tail == head)
return NULL;
/* swing stub node */
_qnode_pool_insert(p, p->stub);
next = tail->next;
if (next) {
p->tail = next;
return tail;
}
return NULL;
}
/*
* This adds a retry to the _pool_remove function
* defined above
*/
static inline struct qnode *__attribute__ ((always_inline))
_qnode_pool_remove(struct qnode_pool *p)
{
struct qnode *n;
do {
n = _pool_remove(p);
if (likely(n != NULL))
return n;
rte_compiler_barrier();
} while ((p->head != p->tail) &&
(p->tail != p->stub));
return NULL;
}
/*
* Allocate a node from the pool
* If the pool is empty add mode nodes
*/
static inline struct qnode *__attribute__ ((always_inline))
_qnode_alloc(void)
{
struct qnode_pool *p = (THIS_SCHED)->qnode_pool;
int prealloc_size = p->pre_alloc;
struct qnode *n;
int i;
if (likely(p->fast_alloc != NULL)) {
n = p->fast_alloc;
p->fast_alloc = NULL;
return n;
}
n = _qnode_pool_remove(p);
if (unlikely(n == NULL)) {
DIAG_COUNT_INC(p, prealloc);
for (i = 0; i < prealloc_size; i++) {
n = rte_malloc_socket(NULL,
sizeof(struct qnode),
RTE_CACHE_LINE_SIZE,
rte_socket_id());
if (n == NULL)
return NULL;
DIAG_COUNT_INC(p, available);
DIAG_COUNT_INC(p, capacity);
n->pool = p;
_qnode_pool_insert(p, n);
}
n = _qnode_pool_remove(p);
}
n->pool = p;
DIAG_COUNT_INC(p, rd);
DIAG_COUNT_DEC(p, available);
return n;
}
/*
* free a queue node to the per scheduler pool from which it came
*/
static inline void __attribute__ ((always_inline))
_qnode_free(struct qnode *n)
{
struct qnode_pool *p = n->pool;
if (unlikely(p->fast_alloc != NULL) ||
unlikely(n->pool != (THIS_SCHED)->qnode_pool)) {
DIAG_COUNT_INC(p, wr);
DIAG_COUNT_INC(p, available);
_qnode_pool_insert(p, n);
return;
}
p->fast_alloc = n;
}
/*
* Destroy an qnode pool
* queue must be empty when this is called
*/
static inline int
_qnode_pool_destroy(struct qnode_pool *p)
{
rte_free(p->stub);
rte_free(p);
return 0;
}
#endif /* LTHREAD_POOL_H_ */

302
examples/performance-thread/common/lthread_queue.h Normal file
View File

@ -0,0 +1,302 @@
/*
*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Some portions of this software is derived from the producer
* consumer queues described by Dmitry Vyukov and published here
* http://www.1024cores.net
*
* Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved.
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DMITRY VYUKOV OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
* OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
* OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* The views and conclusions contained in the software and documentation are
* those of the authors and should not be interpreted as representing official
* policies, either expressed or implied, of Dmitry Vyukov.
*/
#ifndef LTHREAD_QUEUE_H_
#define LTHREAD_QUEUE_H_
#include <string.h>
#include <rte_prefetch.h>
#include <rte_per_lcore.h>
#include "lthread_int.h"
#include "lthread.h"
#include "lthread_diag.h"
#include "lthread_pool.h"
struct lthread_queue;
/*
* This file implements an unbounded FIFO queue based on a lock free
* linked list.
*
* The queue is non-intrusive in that it uses intermediate nodes, and does
* not require these nodes to be inserted into the object being placed
* in the queue.
*
* This is slightly more efficient than the very similar queue in lthread_pool
* in that it does not have to swing a stub node as the queue becomes empty.
*
* The queue access functions allocate and free intermediate node
* transparently from/to a per scheduler pool ( see lthread_pool.h ).
*
* The queue provides both MPSC and SPSC insert methods
*/
/*
* define a queue of lthread nodes
*/
struct lthread_queue {
struct qnode *head;
struct qnode *tail __rte_cache_aligned;
struct lthread_queue *p;
char name[LT_MAX_NAME_SIZE];
DIAG_COUNT_DEFINE(rd);
DIAG_COUNT_DEFINE(wr);
DIAG_COUNT_DEFINE(size);
} __rte_cache_aligned;
static inline struct lthread_queue *
_lthread_queue_create(const char *name)
{
struct qnode *stub;
struct lthread_queue *new_queue;
new_queue = rte_malloc_socket(NULL, sizeof(struct lthread_queue),
RTE_CACHE_LINE_SIZE,
rte_socket_id());
if (new_queue == NULL)
return NULL;
/* allocated stub node */
stub = _qnode_alloc();
LTHREAD_ASSERT(stub);
if (name != NULL)
strncpy(new_queue->name, name, sizeof(new_queue->name));
new_queue->name[sizeof(new_queue->name)-1] = 0;
/* initialize queue as empty */
stub->next = NULL;
new_queue->head = stub;
new_queue->tail = stub;
DIAG_COUNT_INIT(new_queue, rd);
DIAG_COUNT_INIT(new_queue, wr);
DIAG_COUNT_INIT(new_queue, size);
return new_queue;
}
/**
* Return true if the queue is empty
*/
static inline int __attribute__ ((always_inline))
_lthread_queue_empty(struct lthread_queue *q)
{
return (q->tail == q->head);
}
/**
* Destroy a queue
* fail if queue is not empty
*/
static inline int _lthread_queue_destroy(struct lthread_queue *q)
{
if (q == NULL)
return -1;
if (!_lthread_queue_empty(q))
return -1;
_qnode_free(q->head);
rte_free(q);
return 0;
}
RTE_DECLARE_PER_LCORE(struct lthread_sched *, this_sched);
/*
* Insert a node into a queue
* this implementation is multi producer safe
*/
static inline struct qnode *__attribute__ ((always_inline))
_lthread_queue_insert_mp(struct lthread_queue
*q, void *data)
{
struct qnode *prev;
struct qnode *n = _qnode_alloc();
if (n == NULL)
return NULL;
/* set object in node */
n->data = data;
n->next = NULL;
/* this is an MPSC method, perform a locked update */
prev = n;
prev =
(struct qnode *)__sync_lock_test_and_set((uint64_t *) &(q)->head,
(uint64_t) prev);
/* there is a window of inconsistency until prev next is set,
* which is why remove must retry
*/
prev->next = n;
DIAG_COUNT_INC(q, wr);
DIAG_COUNT_INC(q, size);
return n;
}
/*
* Insert an node into a queue in single producer mode
* this implementation is NOT mult producer safe
*/
static inline struct qnode *__attribute__ ((always_inline))
_lthread_queue_insert_sp(struct lthread_queue
*q, void *data)
{
/* allocate a queue node */
struct qnode *prev;
struct qnode *n = _qnode_alloc();
if (n == NULL)
return NULL;
/* set data in node */
n->data = data;
n->next = NULL;
/* this is an SPSC method, no need for locked exchange operation */
prev = q->head;
prev->next = q->head = n;
DIAG_COUNT_INC(q, wr);
DIAG_COUNT_INC(q, size);
return n;
}
/*
* Remove a node from a queue
*/
static inline void *__attribute__ ((always_inline))
_lthread_queue_poll(struct lthread_queue *q)
{
void *data = NULL;
struct qnode *tail = q->tail;
struct qnode *next = (struct qnode *)tail->next;
/*
* There is a small window of inconsistency between producer and
* consumer whereby the queue may appear empty if consumer and
* producer access it at the same time.
* The consumer must handle this by retrying
*/
if (likely(next != NULL)) {
q->tail = next;
tail->data = next->data;
data = tail->data;
/* free the node */
_qnode_free(tail);
DIAG_COUNT_INC(q, rd);
DIAG_COUNT_DEC(q, size);
return data;
}
return NULL;
}
/*
* Remove a node from a queue
*/
static inline void *__attribute__ ((always_inline))
_lthread_queue_remove(struct lthread_queue *q)
{
void *data = NULL;
/*
* There is a small window of inconsistency between producer and
* consumer whereby the queue may appear empty if consumer and
* producer access it at the same time. We handle this by retrying
*/
do {
data = _lthread_queue_poll(q);
if (likely(data != NULL)) {
DIAG_COUNT_INC(q, rd);
DIAG_COUNT_DEC(q, size);
return data;
}
rte_compiler_barrier();
} while (unlikely(!_lthread_queue_empty(q)));
return NULL;
}
#endif /* LTHREAD_QUEUE_H_ */

600
examples/performance-thread/common/lthread_sched.c Normal file
View File

@ -0,0 +1,600 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Some portions of this software is derived from the
* https://github.com/halayli/lthread which carrys the following license.
*
* Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#define RTE_MEM 1
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <stddef.h>
#include <limits.h>
#include <inttypes.h>
#include <unistd.h>
#include <pthread.h>
#include <fcntl.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <sched.h>
#include <rte_config.h>
#include <rte_prefetch.h>
#include <rte_per_lcore.h>
#include <rte_atomic.h>
#include <rte_atomic_64.h>
#include <rte_log.h>
#include <rte_common.h>
#include <rte_branch_prediction.h>
#include "lthread_api.h"
#include "lthread_int.h"
#include "lthread_sched.h"
#include "lthread_objcache.h"
#include "lthread_timer.h"
#include "lthread_mutex.h"
#include "lthread_cond.h"
#include "lthread_tls.h"
#include "lthread_diag.h"
/*
* This file implements the lthread scheduler
* The scheduler is the function lthread_run()
* This must be run as the main loop of an EAL thread.
*
* Currently once a scheduler is created it cannot be destroyed
* When a scheduler shuts down it is assumed that the application is terminating
*/
static rte_atomic16_t num_schedulers;
static rte_atomic16_t active_schedulers;
/* one scheduler per lcore */
RTE_DEFINE_PER_LCORE(struct lthread_sched *, this_sched) = NULL;
struct lthread_sched *schedcore[LTHREAD_MAX_LCORES];
diag_callback diag_cb;
uint64_t diag_mask;
/* constructor */
void lthread_sched_ctor(void) __attribute__ ((constructor));
void lthread_sched_ctor(void)
{
memset(schedcore, 0, sizeof(schedcore));
rte_atomic16_init(&num_schedulers);
rte_atomic16_set(&num_schedulers, 1);
rte_atomic16_init(&active_schedulers);
rte_atomic16_set(&active_schedulers, 0);
diag_cb = NULL;
}
enum sched_alloc_phase {
SCHED_ALLOC_OK,
SCHED_ALLOC_QNODE_POOL,
SCHED_ALLOC_READY_QUEUE,
SCHED_ALLOC_PREADY_QUEUE,
SCHED_ALLOC_LTHREAD_CACHE,
SCHED_ALLOC_STACK_CACHE,
SCHED_ALLOC_PERLT_CACHE,
SCHED_ALLOC_TLS_CACHE,
SCHED_ALLOC_COND_CACHE,
SCHED_ALLOC_MUTEX_CACHE,
};
static int
_lthread_sched_alloc_resources(struct lthread_sched *new_sched)
{
int alloc_status;
do {
/* Initialize per scheduler queue node pool */
alloc_status = SCHED_ALLOC_QNODE_POOL;
new_sched->qnode_pool =
_qnode_pool_create("qnode pool", LTHREAD_PREALLOC);
if (new_sched->qnode_pool == NULL)
break;
/* Initialize per scheduler local ready queue */
alloc_status = SCHED_ALLOC_READY_QUEUE;
new_sched->ready = _lthread_queue_create("ready queue");
if (new_sched->ready == NULL)
break;
/* Initialize per scheduler local peer ready queue */
alloc_status = SCHED_ALLOC_PREADY_QUEUE;
new_sched->pready = _lthread_queue_create("pready queue");
if (new_sched->pready == NULL)
break;
/* Initialize per scheduler local free lthread cache */
alloc_status = SCHED_ALLOC_LTHREAD_CACHE;
new_sched->lthread_cache =
_lthread_objcache_create("lthread cache",
sizeof(struct lthread),
LTHREAD_PREALLOC);
if (new_sched->lthread_cache == NULL)
break;
/* Initialize per scheduler local free stack cache */
alloc_status = SCHED_ALLOC_STACK_CACHE;
new_sched->stack_cache =
_lthread_objcache_create("stack_cache",
sizeof(struct lthread_stack),
LTHREAD_PREALLOC);
if (new_sched->stack_cache == NULL)
break;
/* Initialize per scheduler local free per lthread data cache */
alloc_status = SCHED_ALLOC_PERLT_CACHE;
new_sched->per_lthread_cache =
_lthread_objcache_create("per_lt cache",
RTE_PER_LTHREAD_SECTION_SIZE,
LTHREAD_PREALLOC);
if (new_sched->per_lthread_cache == NULL)
break;
/* Initialize per scheduler local free tls cache */
alloc_status = SCHED_ALLOC_TLS_CACHE;
new_sched->tls_cache =
_lthread_objcache_create("TLS cache",
sizeof(struct lthread_tls),
LTHREAD_PREALLOC);
if (new_sched->tls_cache == NULL)
break;
/* Initialize per scheduler local free cond var cache */
alloc_status = SCHED_ALLOC_COND_CACHE;
new_sched->cond_cache =
_lthread_objcache_create("cond cache",
sizeof(struct lthread_cond),
LTHREAD_PREALLOC);
if (new_sched->cond_cache == NULL)
break;
/* Initialize per scheduler local free mutex cache */
alloc_status = SCHED_ALLOC_MUTEX_CACHE;
new_sched->mutex_cache =
_lthread_objcache_create("mutex cache",
sizeof(struct lthread_mutex),
LTHREAD_PREALLOC);
if (new_sched->mutex_cache == NULL)
break;
alloc_status = SCHED_ALLOC_OK;
} while (0);
/* roll back on any failure */
switch (alloc_status) {
case SCHED_ALLOC_MUTEX_CACHE:
_lthread_objcache_destroy(new_sched->cond_cache);
/* fall through */
case SCHED_ALLOC_COND_CACHE:
_lthread_objcache_destroy(new_sched->tls_cache);
/* fall through */
case SCHED_ALLOC_TLS_CACHE:
_lthread_objcache_destroy(new_sched->per_lthread_cache);
/* fall through */
case SCHED_ALLOC_PERLT_CACHE:
_lthread_objcache_destroy(new_sched->stack_cache);
/* fall through */
case SCHED_ALLOC_STACK_CACHE:
_lthread_objcache_destroy(new_sched->lthread_cache);
/* fall through */
case SCHED_ALLOC_LTHREAD_CACHE:
_lthread_queue_destroy(new_sched->pready);
/* fall through */
case SCHED_ALLOC_PREADY_QUEUE:
_lthread_queue_destroy(new_sched->ready);
/* fall through */
case SCHED_ALLOC_READY_QUEUE:
_qnode_pool_destroy(new_sched->qnode_pool);
/* fall through */
case SCHED_ALLOC_QNODE_POOL:
/* fall through */
case SCHED_ALLOC_OK:
break;
}
return alloc_status;
}
/*
* Create a scheduler on the current lcore
*/
struct lthread_sched *_lthread_sched_create(size_t stack_size)
{
int status;
struct lthread_sched *new_sched;
unsigned lcoreid = rte_lcore_id();
LTHREAD_ASSERT(stack_size <= LTHREAD_MAX_STACK_SIZE);
if (stack_size == 0)
stack_size = LTHREAD_MAX_STACK_SIZE;
new_sched =
rte_calloc_socket(NULL, 1, sizeof(struct lthread_sched),
RTE_CACHE_LINE_SIZE,
rte_socket_id());
if (new_sched == NULL) {
RTE_LOG(CRIT, LTHREAD,
"Failed to allocate memory for scheduler\n");
return NULL;
}
_lthread_key_pool_init();
new_sched->stack_size = stack_size;
new_sched->birth = rte_rdtsc();
THIS_SCHED = new_sched;
status = _lthread_sched_alloc_resources(new_sched);
if (status != SCHED_ALLOC_OK) {
RTE_LOG(CRIT, LTHREAD,
"Failed to allocate resources for scheduler code = %d\n",
status);
rte_free(new_sched);
return NULL;
}
bzero(&new_sched->ctx, sizeof(struct ctx));
new_sched->lcore_id = lcoreid;
schedcore[lcoreid] = new_sched;
new_sched->run_flag = 1;
DIAG_EVENT(new_sched, LT_DIAG_SCHED_CREATE, rte_lcore_id(), 0);
rte_wmb();
return new_sched;
}
/*
* Set the number of schedulers in the system
*/
int lthread_num_schedulers_set(int num)
{
rte_atomic16_set(&num_schedulers, num);
return (int)rte_atomic16_read(&num_schedulers);
}
/*
* Return the number of schedulers active
*/
int lthread_active_schedulers(void)
{
return (int)rte_atomic16_read(&active_schedulers);
}
/**
* shutdown the scheduler running on the specified lcore
*/
void lthread_scheduler_shutdown(unsigned lcoreid)
{
uint64_t coreid = (uint64_t) lcoreid;
if (coreid < LTHREAD_MAX_LCORES) {
if (schedcore[coreid] != NULL)
schedcore[coreid]->run_flag = 0;
}
}
/**
* shutdown all schedulers
*/
void lthread_scheduler_shutdown_all(void)
{
uint64_t i;
/*
* give time for all schedulers to have started
* Note we use sched_yield() rather than pthread_yield() to allow
* for the possibility of a pthread wrapper on lthread_yield(),
* something that is not possible unless the scheduler is running.
*/
while (rte_atomic16_read(&active_schedulers) <
rte_atomic16_read(&num_schedulers))
sched_yield();
for (i = 0; i < LTHREAD_MAX_LCORES; i++) {
if (schedcore[i] != NULL)
schedcore[i]->run_flag = 0;
}
}
/*
* Resume a suspended lthread
*/
static inline void
_lthread_resume(struct lthread *lt) __attribute__ ((always_inline));
static inline void _lthread_resume(struct lthread *lt)
{
struct lthread_sched *sched = THIS_SCHED;
struct lthread_stack *s;
uint64_t state = lt->state;
#if LTHREAD_DIAG
int init = 0;
#endif
sched->current_lthread = lt;
if (state & (BIT(ST_LT_CANCELLED) | BIT(ST_LT_EXITED))) {
/* if detached we can free the thread now */
if (state & BIT(ST_LT_DETACH)) {
_lthread_free(lt);
sched->current_lthread = NULL;
return;
}
}
if (state & BIT(ST_LT_INIT)) {
/* first time this thread has been run */
/* assign thread to this scheduler */
lt->sched = THIS_SCHED;
/* allocate stack */
s = _stack_alloc();
lt->stack_container = s;
_lthread_set_stack(lt, s->stack, s->stack_size);
/* allocate memory for TLS used by this thread */
_lthread_tls_alloc(lt);
lt->state = BIT(ST_LT_READY);
#if LTHREAD_DIAG
init = 1;
#endif
}
DIAG_EVENT(lt, LT_DIAG_LTHREAD_RESUMED, init, lt);
/* switch to the new thread */
ctx_switch(&lt->ctx, &sched->ctx);
/* If posting to a queue that could be read by another lcore
* we defer the queue write till now to ensure the context has been
* saved before the other core tries to resume it
* This applies to blocking on mutex, cond, and to set_affinity
*/
if (lt->pending_wr_queue != NULL) {
struct lthread_queue *dest = lt->pending_wr_queue;
lt->pending_wr_queue = NULL;
/* queue the current thread to the specified queue */
_lthread_queue_insert_mp(dest, lt);
}
sched->current_lthread = NULL;
}
/*
* Handle sleep timer expiry
*/
void
_sched_timer_cb(struct rte_timer *tim, void *arg)
{
struct lthread *lt = (struct lthread *) arg;
uint64_t state = lt->state;
DIAG_EVENT(lt, LT_DIAG_LTHREAD_TMR_EXPIRED, &lt->tim, 0);
rte_timer_stop(tim);
if (lt->state & BIT(ST_LT_CANCELLED))
(THIS_SCHED)->nb_blocked_threads--;
lt->state = state | BIT(ST_LT_EXPIRED);
_lthread_resume(lt);
lt->state = state & CLEARBIT(ST_LT_EXPIRED);
}
/*
* Returns 0 if there is a pending job in scheduler or 1 if done and can exit.
*/
static inline int _lthread_sched_isdone(struct lthread_sched *sched)
{
return ((sched->run_flag == 0) &&
(_lthread_queue_empty(sched->ready)) &&
(_lthread_queue_empty(sched->pready)) &&
(sched->nb_blocked_threads == 0));
}
/*
* Wait for all schedulers to start
*/
static inline void _lthread_schedulers_sync_start(void)
{
rte_atomic16_inc(&active_schedulers);
/* wait for lthread schedulers
* Note we use sched_yield() rather than pthread_yield() to allow
* for the possibility of a pthread wrapper on lthread_yield(),
* something that is not possible unless the scheduler is running.
*/
while (rte_atomic16_read(&active_schedulers) <
rte_atomic16_read(&num_schedulers))
sched_yield();
}
/*
* Wait for all schedulers to stop
*/
static inline void _lthread_schedulers_sync_stop(void)
{
rte_atomic16_dec(&active_schedulers);
rte_atomic16_dec(&num_schedulers);
/* wait for schedulers
* Note we use sched_yield() rather than pthread_yield() to allow
* for the possibility of a pthread wrapper on lthread_yield(),
* something that is not possible unless the scheduler is running.
*/
while (rte_atomic16_read(&active_schedulers) > 0)
sched_yield();
}
/*
* Run the lthread scheduler
* This loop is the heart of the system
*/
void lthread_run(void)
{
struct lthread_sched *sched = THIS_SCHED;
struct lthread *lt = NULL;
RTE_LOG(INFO, LTHREAD,
"starting scheduler %p on lcore %u phys core %u\n",
sched, rte_lcore_id(),
rte_lcore_index(rte_lcore_id()));
/* if more than one, wait for all schedulers to start */
_lthread_schedulers_sync_start();
/*
* This is the main scheduling loop
* So long as there are tasks in existence we run this loop.
* We check for:-
* expired timers,
* the local ready queue,
* and the peer ready queue,
*
* and resume lthreads ad infinitum.
*/
while (!_lthread_sched_isdone(sched)) {
rte_timer_manage();
lt = _lthread_queue_poll(sched->ready);
if (lt != NULL)
_lthread_resume(lt);
lt = _lthread_queue_poll(sched->pready);
if (lt != NULL)
_lthread_resume(lt);
}
/* if more than one wait for all schedulers to stop */
_lthread_schedulers_sync_stop();
(THIS_SCHED) = NULL;
RTE_LOG(INFO, LTHREAD,
"stopping scheduler %p on lcore %u phys core %u\n",
sched, rte_lcore_id(),
rte_lcore_index(rte_lcore_id()));
fflush(stdout);
}
/*
* Return the scheduler for this lcore
*
*/
struct lthread_sched *_lthread_sched_get(int lcore_id)
{
if (lcore_id > LTHREAD_MAX_LCORES)
return NULL;
return schedcore[lcore_id];
}
/*
* migrate the current thread to another scheduler running
* on the specified lcore.
*/
int lthread_set_affinity(unsigned lcoreid)
{
struct lthread *lt = THIS_LTHREAD;
struct lthread_sched *dest_sched;
if (unlikely(lcoreid > LTHREAD_MAX_LCORES))
return POSIX_ERRNO(EINVAL);
DIAG_EVENT(lt, LT_DIAG_LTHREAD_AFFINITY, lcoreid, 0);
dest_sched = schedcore[lcoreid];
if (unlikely(dest_sched == NULL))
return POSIX_ERRNO(EINVAL);
if (likely(dest_sched != THIS_SCHED)) {
lt->sched = dest_sched;
lt->pending_wr_queue = dest_sched->pready;
_affinitize();
return 0;
}
return 0;
}

152
examples/performance-thread/common/lthread_sched.h Normal file
View File

@ -0,0 +1,152 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Some portions of this software is derived from the
* https://github.com/halayli/lthread which carrys the following license.
*
* Copyright (C) 2012, Hasan Alayli <halayli@gmail.com>
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef LTHREAD_SCHED_H_
#define LTHREAD_SCHED_H_
#include "lthread_int.h"
#include "lthread_queue.h"
#include "lthread_objcache.h"
#include "lthread_diag.h"
#include "ctx.h"
/*
* insert an lthread into a queue
*/
static inline void
_ready_queue_insert(struct lthread_sched *sched, struct lthread *lt)
{
if (sched == THIS_SCHED)
_lthread_queue_insert_sp((THIS_SCHED)->ready, lt);
else
_lthread_queue_insert_mp(sched->pready, lt);
}
/*
* remove an lthread from a queue
*/
static inline struct lthread *_ready_queue_remove(struct lthread_queue *q)
{
return _lthread_queue_remove(q);
}
/**
* Return true if the ready queue is empty
*/
static inline int _ready_queue_empty(struct lthread_queue *q)
{
return _lthread_queue_empty(q);
}
static inline uint64_t _sched_now(void)
{
uint64_t now = rte_rdtsc();
if (now > (THIS_SCHED)->birth)
return now - (THIS_SCHED)->birth;
if (now < (THIS_SCHED)->birth)
return (THIS_SCHED)->birth - now;
/* never return 0 because this means sleep forever */
return 1;
}
static inline void
_affinitize(void) __attribute__ ((always_inline));
static inline void
_affinitize(void)
{
struct lthread *lt = THIS_LTHREAD;
DIAG_EVENT(lt, LT_DIAG_LTHREAD_SUSPENDED, 0, 0);
ctx_switch(&(THIS_SCHED)->ctx, &lt->ctx);
}
static inline void
_suspend(void) __attribute__ ((always_inline));
static inline void
_suspend(void)
{
struct lthread *lt = THIS_LTHREAD;
(THIS_SCHED)->nb_blocked_threads++;
DIAG_EVENT(lt, LT_DIAG_LTHREAD_SUSPENDED, 0, 0);
ctx_switch(&(THIS_SCHED)->ctx, &lt->ctx);
(THIS_SCHED)->nb_blocked_threads--;
}
static inline void
_reschedule(void) __attribute__ ((always_inline));
static inline void
_reschedule(void)
{
struct lthread *lt = THIS_LTHREAD;
DIAG_EVENT(lt, LT_DIAG_LTHREAD_RESCHEDULED, 0, 0);
_ready_queue_insert(THIS_SCHED, lt);
ctx_switch(&(THIS_SCHED)->ctx, &lt->ctx);
}
extern struct lthread_sched *schedcore[];
void _sched_timer_cb(struct rte_timer *tim, void *arg);
void _sched_shutdown(__rte_unused void *arg);
#endif /* LTHREAD_SCHED_H_ */

79
examples/performance-thread/common/lthread_timer.h Normal file
View File

@ -0,0 +1,79 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef LTHREAD_TIMER_H_
#define LTHREAD_TIMER_H_
#include "lthread_int.h"
#include "lthread_sched.h"
static inline uint64_t
_ns_to_clks(uint64_t ns)
{
unsigned __int128 clkns = rte_get_tsc_hz();
clkns *= ns;
clkns /= 1000000000;
return (uint64_t) clkns;
}
static inline void
_timer_start(struct lthread *lt, uint64_t clks)
{
if (clks > 0) {
DIAG_EVENT(lt, LT_DIAG_LTHREAD_TMR_START, &lt->tim, clks);
rte_timer_init(&lt->tim);
rte_timer_reset(&lt->tim,
clks,
SINGLE,
rte_lcore_id(),
_sched_timer_cb,
(void *)lt);
}
}
static inline void
_timer_stop(struct lthread *lt)
{
if (lt != NULL) {
DIAG_EVENT(lt, LT_DIAG_LTHREAD_TMR_DELETE, &lt->tim, 0);
rte_timer_stop(&lt->tim);
}
}
#endif /* LTHREAD_TIMER_H_ */

254
examples/performance-thread/common/lthread_tls.c Normal file
View File

@ -0,0 +1,254 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <limits.h>
#include <inttypes.h>
#include <unistd.h>
#include <pthread.h>
#include <fcntl.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <execinfo.h>
#include <sched.h>
#include <rte_config.h>
#include <rte_malloc.h>
#include <rte_log.h>
#include <rte_ring.h>
#include <rte_atomic_64.h>
#include "lthread_tls.h"
#include "lthread_queue.h"
#include "lthread_objcache.h"
#include "lthread_sched.h"
static struct rte_ring *key_pool;
static uint64_t key_pool_init;
/* needed to cause section start and end to be defined */
RTE_DEFINE_PER_LTHREAD(void *, dummy);
static struct lthread_key key_table[LTHREAD_MAX_KEYS];
void lthread_tls_ctor(void) __attribute__((constructor));
void lthread_tls_ctor(void)
{
key_pool = NULL;
key_pool_init = 0;
}
/*
* Initialize a pool of keys
* These are unique tokens that can be obtained by threads
* calling lthread_key_create()
*/
void _lthread_key_pool_init(void)
{
static struct rte_ring *pool;
struct lthread_key *new_key;
char name[MAX_LTHREAD_NAME_SIZE];
bzero(key_table, sizeof(key_table));
/* only one lcore should do this */
if (rte_atomic64_cmpset(&key_pool_init, 0, 1)) {
snprintf(name,
MAX_LTHREAD_NAME_SIZE,
"lthread_key_pool_%d",
getpid());
pool = rte_ring_create(name,
LTHREAD_MAX_KEYS, 0, 0);
LTHREAD_ASSERT(pool);
int i;
for (i = 1; i < LTHREAD_MAX_KEYS; i++) {
new_key = &key_table[i];
rte_ring_mp_enqueue((struct rte_ring *)pool,
(void *)new_key);
}
key_pool = pool;
}
/* other lcores wait here till done */
while (key_pool == NULL) {
rte_compiler_barrier();
sched_yield();
};
}
/*
* Create a key
* this means getting a key from the the pool
*/
int lthread_key_create(unsigned int *key, tls_destructor_func destructor)
{
if (key == NULL)
return POSIX_ERRNO(EINVAL);
struct lthread_key *new_key;
if (rte_ring_mc_dequeue((struct rte_ring *)key_pool, (void **)&new_key)
== 0) {
new_key->destructor = destructor;
*key = (new_key - key_table);
return 0;
}
return POSIX_ERRNO(EAGAIN);
}
/*
* Delete a key
*/
int lthread_key_delete(unsigned int k)
{
struct lthread_key *key;
key = (struct lthread_key *) &key_table[k];
if (k > LTHREAD_MAX_KEYS)
return POSIX_ERRNO(EINVAL);
key->destructor = NULL;
rte_ring_mp_enqueue((struct rte_ring *)key_pool,
(void *)key);
return 0;
}
/*
* Break association for all keys in use by this thread
* invoke the destructor if available.
* Since a destructor can create keys we could enter an infinite loop
* therefore we give up after LTHREAD_DESTRUCTOR_ITERATIONS
* the behavior is modelled on pthread
*/
void _lthread_tls_destroy(struct lthread *lt)
{
int i, k;
int nb_keys;
void *data;
for (i = 0; i < LTHREAD_DESTRUCTOR_ITERATIONS; i++) {
for (k = 1; k < LTHREAD_MAX_KEYS; k++) {
/* no keys in use ? */
nb_keys = lt->tls->nb_keys_inuse;
if (nb_keys == 0)
return;
/* this key not in use ? */
if (lt->tls->data[k] == NULL)
continue;
/* remove this key */
data = lt->tls->data[k];
lt->tls->data[k] = NULL;
lt->tls->nb_keys_inuse = nb_keys-1;
/* invoke destructor */
if (key_table[k].destructor != NULL)
key_table[k].destructor(data);
}
}
}
/*
* Return the pointer associated with a key
* If the key is no longer valid return NULL
*/
void
*lthread_getspecific(unsigned int k)
{
if (k > LTHREAD_MAX_KEYS)
return NULL;
return THIS_LTHREAD->tls->data[k];
}
/*
* Set a value against a key
* If the key is no longer valid return an error
* when storing value
*/
int lthread_setspecific(unsigned int k, const void *data)
{
if (k > LTHREAD_MAX_KEYS)
return POSIX_ERRNO(EINVAL);
int n = THIS_LTHREAD->tls->nb_keys_inuse;
/* discard const qualifier */
char *p = (char *) (uintptr_t) data;
if (data != NULL) {
if (THIS_LTHREAD->tls->data[k] == NULL)
THIS_LTHREAD->tls->nb_keys_inuse = n+1;
}
THIS_LTHREAD->tls->data[k] = (void *) p;
return 0;
}
/*
* Allocate data for TLS cache
*/
void _lthread_tls_alloc(struct lthread *lt)
{
struct lthread_tls *tls;
tls = _lthread_objcache_alloc((THIS_SCHED)->tls_cache);
LTHREAD_ASSERT(tls != NULL);
tls->root_sched = (THIS_SCHED);
lt->tls = tls;
/* allocate data for TLS varaiables using RTE_PER_LTHREAD macros */
if (sizeof(void *) < (uint64_t)RTE_PER_LTHREAD_SECTION_SIZE) {
lt->per_lthread_data =
_lthread_objcache_alloc((THIS_SCHED)->per_lthread_cache);
}
}

57
examples/performance-thread/common/lthread_tls.h Normal file
View File

@ -0,0 +1,57 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2015 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef LTHREAD_TLS_H_
#define LTHREAD_TLS_H_
#include "lthread_api.h"
#define RTE_PER_LTHREAD_SECTION_SIZE \
(&__stop_per_lt - &__start_per_lt)
struct lthread_key {
tls_destructor_func destructor;
};
struct lthread_tls {
void *data[LTHREAD_MAX_KEYS];
int nb_keys_inuse;
struct lthread_sched *root_sched;
};
void _lthread_tls_destroy(struct lthread *lt);
void _lthread_key_pool_init(void);
void _lthread_tls_alloc(struct lthread *lt);
#endif /* LTHREAD_TLS_H_ */