2019-04-03 23:20:13 +00:00
|
|
|
/* SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
|
* Copyright(c) 2019 Intel Corporation
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include <string.h>
|
|
|
|
|
|
|
|
|
|
#include <rte_atomic.h>
|
|
|
|
|
#include <rte_eal.h>
|
|
|
|
|
#include <rte_eal_memconfig.h>
|
|
|
|
|
#include <rte_errno.h>
|
|
|
|
|
#include <rte_malloc.h>
|
|
|
|
|
#include <rte_memzone.h>
|
|
|
|
|
#include <rte_rwlock.h>
|
|
|
|
|
#include <rte_tailq.h>
|
|
|
|
|
|
|
|
|
|
#include "rte_stack.h"
|
|
|
|
|
#include "rte_stack_pvt.h"
|
|
|
|
|
|
|
|
|
|
int stack_logtype;
|
|
|
|
|
|
|
|
|
|
TAILQ_HEAD(rte_stack_list, rte_tailq_entry);
|
|
|
|
|
|
|
|
|
|
static struct rte_tailq_elem rte_stack_tailq = {
|
|
|
|
|
.name = RTE_TAILQ_STACK_NAME,
|
|
|
|
|
};
|
|
|
|
|
EAL_REGISTER_TAILQ(rte_stack_tailq)
|
|
|
|
|
|
stack: add lock-free implementation
This commit adds support for a lock-free (linked list based) stack to the
stack API. This behavior is selected through a new rte_stack_create() flag,
RTE_STACK_F_LF.
The stack consists of a linked list of elements, each containing a data
pointer and a next pointer, and an atomic stack depth counter.
The lock-free push operation enqueues a linked list of pointers by pointing
the tail of the list to the current stack head, and using a CAS to swing
the stack head pointer to the head of the list. The operation retries if it
is unsuccessful (i.e. the list changed between reading the head and
modifying it), else it adjusts the stack length and returns.
The lock-free pop operation first reserves num elements by adjusting the
stack length, to ensure the dequeue operation will succeed without
blocking. It then dequeues pointers by walking the list -- starting from
the head -- then swinging the head pointer (using a CAS as well). While
walking the list, the data pointers are recorded in an object table.
This algorithm stack uses a 128-bit compare-and-swap instruction, which
atomically updates the stack top pointer and a modification counter, to
protect against the ABA problem.
The linked list elements themselves are maintained in a lock-free LIFO
list, and are allocated before stack pushes and freed after stack pops.
Since the stack has a fixed maximum depth, these elements do not need to be
dynamically created.
Signed-off-by: Gage Eads <gage.eads@intel.com>
Reviewed-by: Olivier Matz <olivier.matz@6wind.com>
Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
2019-04-03 23:20:17 +00:00
|
|
|
|
2019-04-03 23:20:13 +00:00
|
|
|
static void
|
stack: add lock-free implementation
This commit adds support for a lock-free (linked list based) stack to the
stack API. This behavior is selected through a new rte_stack_create() flag,
RTE_STACK_F_LF.
The stack consists of a linked list of elements, each containing a data
pointer and a next pointer, and an atomic stack depth counter.
The lock-free push operation enqueues a linked list of pointers by pointing
the tail of the list to the current stack head, and using a CAS to swing
the stack head pointer to the head of the list. The operation retries if it
is unsuccessful (i.e. the list changed between reading the head and
modifying it), else it adjusts the stack length and returns.
The lock-free pop operation first reserves num elements by adjusting the
stack length, to ensure the dequeue operation will succeed without
blocking. It then dequeues pointers by walking the list -- starting from
the head -- then swinging the head pointer (using a CAS as well). While
walking the list, the data pointers are recorded in an object table.
This algorithm stack uses a 128-bit compare-and-swap instruction, which
atomically updates the stack top pointer and a modification counter, to
protect against the ABA problem.
The linked list elements themselves are maintained in a lock-free LIFO
list, and are allocated before stack pushes and freed after stack pops.
Since the stack has a fixed maximum depth, these elements do not need to be
dynamically created.
Signed-off-by: Gage Eads <gage.eads@intel.com>
Reviewed-by: Olivier Matz <olivier.matz@6wind.com>
Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
2019-04-03 23:20:17 +00:00
|
|
|
rte_stack_init(struct rte_stack *s, unsigned int count, uint32_t flags)
|
2019-04-03 23:20:13 +00:00
|
|
|
{
|
|
|
|
|
memset(s, 0, sizeof(*s));
|
|
|
|
|
|
stack: add lock-free implementation
This commit adds support for a lock-free (linked list based) stack to the
stack API. This behavior is selected through a new rte_stack_create() flag,
RTE_STACK_F_LF.
The stack consists of a linked list of elements, each containing a data
pointer and a next pointer, and an atomic stack depth counter.
The lock-free push operation enqueues a linked list of pointers by pointing
the tail of the list to the current stack head, and using a CAS to swing
the stack head pointer to the head of the list. The operation retries if it
is unsuccessful (i.e. the list changed between reading the head and
modifying it), else it adjusts the stack length and returns.
The lock-free pop operation first reserves num elements by adjusting the
stack length, to ensure the dequeue operation will succeed without
blocking. It then dequeues pointers by walking the list -- starting from
the head -- then swinging the head pointer (using a CAS as well). While
walking the list, the data pointers are recorded in an object table.
This algorithm stack uses a 128-bit compare-and-swap instruction, which
atomically updates the stack top pointer and a modification counter, to
protect against the ABA problem.
The linked list elements themselves are maintained in a lock-free LIFO
list, and are allocated before stack pushes and freed after stack pops.
Since the stack has a fixed maximum depth, these elements do not need to be
dynamically created.
Signed-off-by: Gage Eads <gage.eads@intel.com>
Reviewed-by: Olivier Matz <olivier.matz@6wind.com>
Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
2019-04-03 23:20:17 +00:00
|
|
|
if (flags & RTE_STACK_F_LF)
|
|
|
|
|
rte_stack_lf_init(s, count);
|
|
|
|
|
else
|
|
|
|
|
rte_stack_std_init(s);
|
2019-04-03 23:20:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static ssize_t
|
stack: add lock-free implementation
This commit adds support for a lock-free (linked list based) stack to the
stack API. This behavior is selected through a new rte_stack_create() flag,
RTE_STACK_F_LF.
The stack consists of a linked list of elements, each containing a data
pointer and a next pointer, and an atomic stack depth counter.
The lock-free push operation enqueues a linked list of pointers by pointing
the tail of the list to the current stack head, and using a CAS to swing
the stack head pointer to the head of the list. The operation retries if it
is unsuccessful (i.e. the list changed between reading the head and
modifying it), else it adjusts the stack length and returns.
The lock-free pop operation first reserves num elements by adjusting the
stack length, to ensure the dequeue operation will succeed without
blocking. It then dequeues pointers by walking the list -- starting from
the head -- then swinging the head pointer (using a CAS as well). While
walking the list, the data pointers are recorded in an object table.
This algorithm stack uses a 128-bit compare-and-swap instruction, which
atomically updates the stack top pointer and a modification counter, to
protect against the ABA problem.
The linked list elements themselves are maintained in a lock-free LIFO
list, and are allocated before stack pushes and freed after stack pops.
Since the stack has a fixed maximum depth, these elements do not need to be
dynamically created.
Signed-off-by: Gage Eads <gage.eads@intel.com>
Reviewed-by: Olivier Matz <olivier.matz@6wind.com>
Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
2019-04-03 23:20:17 +00:00
|
|
|
rte_stack_get_memsize(unsigned int count, uint32_t flags)
|
2019-04-03 23:20:13 +00:00
|
|
|
{
|
stack: add lock-free implementation
This commit adds support for a lock-free (linked list based) stack to the
stack API. This behavior is selected through a new rte_stack_create() flag,
RTE_STACK_F_LF.
The stack consists of a linked list of elements, each containing a data
pointer and a next pointer, and an atomic stack depth counter.
The lock-free push operation enqueues a linked list of pointers by pointing
the tail of the list to the current stack head, and using a CAS to swing
the stack head pointer to the head of the list. The operation retries if it
is unsuccessful (i.e. the list changed between reading the head and
modifying it), else it adjusts the stack length and returns.
The lock-free pop operation first reserves num elements by adjusting the
stack length, to ensure the dequeue operation will succeed without
blocking. It then dequeues pointers by walking the list -- starting from
the head -- then swinging the head pointer (using a CAS as well). While
walking the list, the data pointers are recorded in an object table.
This algorithm stack uses a 128-bit compare-and-swap instruction, which
atomically updates the stack top pointer and a modification counter, to
protect against the ABA problem.
The linked list elements themselves are maintained in a lock-free LIFO
list, and are allocated before stack pushes and freed after stack pops.
Since the stack has a fixed maximum depth, these elements do not need to be
dynamically created.
Signed-off-by: Gage Eads <gage.eads@intel.com>
Reviewed-by: Olivier Matz <olivier.matz@6wind.com>
Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
2019-04-03 23:20:17 +00:00
|
|
|
if (flags & RTE_STACK_F_LF)
|
|
|
|
|
return rte_stack_lf_get_memsize(count);
|
|
|
|
|
else
|
|
|
|
|
return rte_stack_std_get_memsize(count);
|
2019-04-03 23:20:13 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct rte_stack *
|
|
|
|
|
rte_stack_create(const char *name, unsigned int count, int socket_id,
|
|
|
|
|
uint32_t flags)
|
|
|
|
|
{
|
|
|
|
|
char mz_name[RTE_MEMZONE_NAMESIZE];
|
|
|
|
|
struct rte_stack_list *stack_list;
|
|
|
|
|
const struct rte_memzone *mz;
|
|
|
|
|
struct rte_tailq_entry *te;
|
|
|
|
|
struct rte_stack *s;
|
|
|
|
|
unsigned int sz;
|
|
|
|
|
int ret;
|
|
|
|
|
|
stack: add lock-free implementation
This commit adds support for a lock-free (linked list based) stack to the
stack API. This behavior is selected through a new rte_stack_create() flag,
RTE_STACK_F_LF.
The stack consists of a linked list of elements, each containing a data
pointer and a next pointer, and an atomic stack depth counter.
The lock-free push operation enqueues a linked list of pointers by pointing
the tail of the list to the current stack head, and using a CAS to swing
the stack head pointer to the head of the list. The operation retries if it
is unsuccessful (i.e. the list changed between reading the head and
modifying it), else it adjusts the stack length and returns.
The lock-free pop operation first reserves num elements by adjusting the
stack length, to ensure the dequeue operation will succeed without
blocking. It then dequeues pointers by walking the list -- starting from
the head -- then swinging the head pointer (using a CAS as well). While
walking the list, the data pointers are recorded in an object table.
This algorithm stack uses a 128-bit compare-and-swap instruction, which
atomically updates the stack top pointer and a modification counter, to
protect against the ABA problem.
The linked list elements themselves are maintained in a lock-free LIFO
list, and are allocated before stack pushes and freed after stack pops.
Since the stack has a fixed maximum depth, these elements do not need to be
dynamically created.
Signed-off-by: Gage Eads <gage.eads@intel.com>
Reviewed-by: Olivier Matz <olivier.matz@6wind.com>
Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
2019-04-03 23:20:17 +00:00
|
|
|
#ifdef RTE_ARCH_64
|
|
|
|
|
RTE_BUILD_BUG_ON(sizeof(struct rte_stack_lf_head) != 16);
|
|
|
|
|
#else
|
|
|
|
|
if (flags & RTE_STACK_F_LF) {
|
|
|
|
|
STACK_LOG_ERR("Lock-free stack is not supported on your platform\n");
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
2019-04-03 23:20:13 +00:00
|
|
|
|
stack: add lock-free implementation
This commit adds support for a lock-free (linked list based) stack to the
stack API. This behavior is selected through a new rte_stack_create() flag,
RTE_STACK_F_LF.
The stack consists of a linked list of elements, each containing a data
pointer and a next pointer, and an atomic stack depth counter.
The lock-free push operation enqueues a linked list of pointers by pointing
the tail of the list to the current stack head, and using a CAS to swing
the stack head pointer to the head of the list. The operation retries if it
is unsuccessful (i.e. the list changed between reading the head and
modifying it), else it adjusts the stack length and returns.
The lock-free pop operation first reserves num elements by adjusting the
stack length, to ensure the dequeue operation will succeed without
blocking. It then dequeues pointers by walking the list -- starting from
the head -- then swinging the head pointer (using a CAS as well). While
walking the list, the data pointers are recorded in an object table.
This algorithm stack uses a 128-bit compare-and-swap instruction, which
atomically updates the stack top pointer and a modification counter, to
protect against the ABA problem.
The linked list elements themselves are maintained in a lock-free LIFO
list, and are allocated before stack pushes and freed after stack pops.
Since the stack has a fixed maximum depth, these elements do not need to be
dynamically created.
Signed-off-by: Gage Eads <gage.eads@intel.com>
Reviewed-by: Olivier Matz <olivier.matz@6wind.com>
Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
2019-04-03 23:20:17 +00:00
|
|
|
sz = rte_stack_get_memsize(count, flags);
|
2019-04-03 23:20:13 +00:00
|
|
|
|
|
|
|
|
ret = snprintf(mz_name, sizeof(mz_name), "%s%s",
|
|
|
|
|
RTE_STACK_MZ_PREFIX, name);
|
|
|
|
|
if (ret < 0 || ret >= (int)sizeof(mz_name)) {
|
|
|
|
|
rte_errno = ENAMETOOLONG;
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
te = rte_zmalloc("STACK_TAILQ_ENTRY", sizeof(*te), 0);
|
|
|
|
|
if (te == NULL) {
|
|
|
|
|
STACK_LOG_ERR("Cannot reserve memory for tailq\n");
|
|
|
|
|
rte_errno = ENOMEM;
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
|
|
|
|
|
|
|
|
|
|
mz = rte_memzone_reserve_aligned(mz_name, sz, socket_id,
|
|
|
|
|
0, __alignof__(*s));
|
|
|
|
|
if (mz == NULL) {
|
|
|
|
|
STACK_LOG_ERR("Cannot reserve stack memzone!\n");
|
|
|
|
|
rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
|
|
|
|
|
rte_free(te);
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
s = mz->addr;
|
|
|
|
|
|
stack: add lock-free implementation
This commit adds support for a lock-free (linked list based) stack to the
stack API. This behavior is selected through a new rte_stack_create() flag,
RTE_STACK_F_LF.
The stack consists of a linked list of elements, each containing a data
pointer and a next pointer, and an atomic stack depth counter.
The lock-free push operation enqueues a linked list of pointers by pointing
the tail of the list to the current stack head, and using a CAS to swing
the stack head pointer to the head of the list. The operation retries if it
is unsuccessful (i.e. the list changed between reading the head and
modifying it), else it adjusts the stack length and returns.
The lock-free pop operation first reserves num elements by adjusting the
stack length, to ensure the dequeue operation will succeed without
blocking. It then dequeues pointers by walking the list -- starting from
the head -- then swinging the head pointer (using a CAS as well). While
walking the list, the data pointers are recorded in an object table.
This algorithm stack uses a 128-bit compare-and-swap instruction, which
atomically updates the stack top pointer and a modification counter, to
protect against the ABA problem.
The linked list elements themselves are maintained in a lock-free LIFO
list, and are allocated before stack pushes and freed after stack pops.
Since the stack has a fixed maximum depth, these elements do not need to be
dynamically created.
Signed-off-by: Gage Eads <gage.eads@intel.com>
Reviewed-by: Olivier Matz <olivier.matz@6wind.com>
Reviewed-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
2019-04-03 23:20:17 +00:00
|
|
|
rte_stack_init(s, count, flags);
|
2019-04-03 23:20:13 +00:00
|
|
|
|
|
|
|
|
/* Store the name for later lookups */
|
|
|
|
|
ret = snprintf(s->name, sizeof(s->name), "%s", name);
|
|
|
|
|
if (ret < 0 || ret >= (int)sizeof(s->name)) {
|
|
|
|
|
rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
|
|
|
|
|
|
|
|
|
|
rte_errno = ENAMETOOLONG;
|
|
|
|
|
rte_free(te);
|
|
|
|
|
rte_memzone_free(mz);
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
s->memzone = mz;
|
|
|
|
|
s->capacity = count;
|
|
|
|
|
s->flags = flags;
|
|
|
|
|
|
|
|
|
|
te->data = s;
|
|
|
|
|
|
|
|
|
|
stack_list = RTE_TAILQ_CAST(rte_stack_tailq.head, rte_stack_list);
|
|
|
|
|
|
|
|
|
|
TAILQ_INSERT_TAIL(stack_list, te, next);
|
|
|
|
|
|
|
|
|
|
rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
|
|
|
|
|
|
|
|
|
|
return s;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void
|
|
|
|
|
rte_stack_free(struct rte_stack *s)
|
|
|
|
|
{
|
|
|
|
|
struct rte_stack_list *stack_list;
|
|
|
|
|
struct rte_tailq_entry *te;
|
|
|
|
|
|
|
|
|
|
if (s == NULL)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
stack_list = RTE_TAILQ_CAST(rte_stack_tailq.head, rte_stack_list);
|
|
|
|
|
rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK);
|
|
|
|
|
|
|
|
|
|
/* find out tailq entry */
|
|
|
|
|
TAILQ_FOREACH(te, stack_list, next) {
|
|
|
|
|
if (te->data == s)
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (te == NULL) {
|
|
|
|
|
rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
TAILQ_REMOVE(stack_list, te, next);
|
|
|
|
|
|
|
|
|
|
rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK);
|
|
|
|
|
|
|
|
|
|
rte_free(te);
|
|
|
|
|
|
|
|
|
|
rte_memzone_free(s->memzone);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct rte_stack *
|
|
|
|
|
rte_stack_lookup(const char *name)
|
|
|
|
|
{
|
|
|
|
|
struct rte_stack_list *stack_list;
|
|
|
|
|
struct rte_tailq_entry *te;
|
|
|
|
|
struct rte_stack *r = NULL;
|
|
|
|
|
|
|
|
|
|
if (name == NULL) {
|
|
|
|
|
rte_errno = EINVAL;
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
stack_list = RTE_TAILQ_CAST(rte_stack_tailq.head, rte_stack_list);
|
|
|
|
|
|
|
|
|
|
rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK);
|
|
|
|
|
|
|
|
|
|
TAILQ_FOREACH(te, stack_list, next) {
|
|
|
|
|
r = (struct rte_stack *) te->data;
|
|
|
|
|
if (strncmp(name, r->name, RTE_STACK_NAMESIZE) == 0)
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK);
|
|
|
|
|
|
|
|
|
|
if (te == NULL) {
|
|
|
|
|
rte_errno = ENOENT;
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return r;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
RTE_INIT(librte_stack_init_log)
|
|
|
|
|
{
|
|
|
|
|
stack_logtype = rte_log_register("lib.stack");
|
|
|
|
|
if (stack_logtype >= 0)
|
|
|
|
|
rte_log_set_level(stack_logtype, RTE_LOG_NOTICE);
|
|
|
|
|
}
|