d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
3a19eeb0c3
numam-dpdk/drivers/net/mlx5/mlx5_utils.c

485 lines
12 KiB
C
Raw Normal View History

net/mlx5: introduce hash list Introduce simple hash list to the mlx5 utilities. User can define its own data structure containing the mlx5_hlist_entry and create the hash list table via the creation interface. Then the entry will be inserted into the table and linked to the corresponding list head. User should guarantee there is no collision of the key and provide a callback function to handle all the remaining entries in the table when destroying the hash list. User should define a proper number of the list heads in the table in order to get a better performance. The LSB of the 'key' is used to calculate the index of the head in the list heads array. This implementation is not multi-threads safe right now. Signed-off-by: Bing Zhao <bingz@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2019-11-06 13:07:56 +00:00
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright 2019 Mellanox Technologies, Ltd
*/
#include <rte_malloc.h>
#include <rte_hash_crc.h>
#include "mlx5_utils.h"
struct mlx5_hlist *
mlx5_hlist_create(const char *name, uint32_t size)
{
struct mlx5_hlist *h;
uint32_t act_size;
uint32_t alloc_size;
if (!size)
return NULL;
/* Align to the next power of 2, 32bits integer is enough now. */
if (!rte_is_power_of_2(size)) {
act_size = rte_align32pow2(size);
DRV_LOG(WARNING, "Size 0x%" PRIX32 " is not power of 2, will "
"be aligned to 0x%" PRIX32 ".\n", size, act_size);
} else {
act_size = size;
}
alloc_size = sizeof(struct mlx5_hlist) +
sizeof(struct mlx5_hlist_head) * act_size;
/* Using zmalloc, then no need to initialize the heads. */
h = rte_zmalloc(name, alloc_size, RTE_CACHE_LINE_SIZE);
if (!h) {
DRV_LOG(ERR, "No memory for hash list %s creation\n",
name ? name : "None");
return NULL;
}
if (name)
snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name);
h->table_sz = act_size;
h->mask = act_size - 1;
DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.\n",
h->name, act_size);
return h;
}
struct mlx5_hlist_entry *
mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key)
{
uint32_t idx;
struct mlx5_hlist_head *first;
struct mlx5_hlist_entry *node;
common/mlx5: improve assert control Use the MLX5_ASSERT macros instead of the standard assert clause. Depends on the RTE_LIBRTE_MLX5_DEBUG configuration option to define it. If RTE_LIBRTE_MLX5_DEBUG is enabled MLX5_ASSERT is equal to RTE_VERIFY to bypass the global CONFIG_RTE_ENABLE_ASSERT option. If RTE_LIBRTE_MLX5_DEBUG is disabled, the global CONFIG_RTE_ENABLE_ASSERT can still make this assert active by calling RTE_VERIFY inside RTE_ASSERT. Signed-off-by: Alexander Kozyrev <akozyrev@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-01-30 16:14:40 +00:00
MLX5_ASSERT(h);
net/mlx5: introduce hash list Introduce simple hash list to the mlx5 utilities. User can define its own data structure containing the mlx5_hlist_entry and create the hash list table via the creation interface. Then the entry will be inserted into the table and linked to the corresponding list head. User should guarantee there is no collision of the key and provide a callback function to handle all the remaining entries in the table when destroying the hash list. User should define a proper number of the list heads in the table in order to get a better performance. The LSB of the 'key' is used to calculate the index of the head in the list heads array. This implementation is not multi-threads safe right now. Signed-off-by: Bing Zhao <bingz@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2019-11-06 13:07:56 +00:00
idx = rte_hash_crc_8byte(key, 0) & h->mask;
first = &h->heads[idx];
LIST_FOREACH(node, first, next) {
if (node->key == key)
return node;
}
return NULL;
}
int
mlx5_hlist_insert(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry)
{
uint32_t idx;
struct mlx5_hlist_head *first;
struct mlx5_hlist_entry *node;
common/mlx5: improve assert control Use the MLX5_ASSERT macros instead of the standard assert clause. Depends on the RTE_LIBRTE_MLX5_DEBUG configuration option to define it. If RTE_LIBRTE_MLX5_DEBUG is enabled MLX5_ASSERT is equal to RTE_VERIFY to bypass the global CONFIG_RTE_ENABLE_ASSERT option. If RTE_LIBRTE_MLX5_DEBUG is disabled, the global CONFIG_RTE_ENABLE_ASSERT can still make this assert active by calling RTE_VERIFY inside RTE_ASSERT. Signed-off-by: Alexander Kozyrev <akozyrev@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-01-30 16:14:40 +00:00
MLX5_ASSERT(h && entry);
net/mlx5: introduce hash list Introduce simple hash list to the mlx5 utilities. User can define its own data structure containing the mlx5_hlist_entry and create the hash list table via the creation interface. Then the entry will be inserted into the table and linked to the corresponding list head. User should guarantee there is no collision of the key and provide a callback function to handle all the remaining entries in the table when destroying the hash list. User should define a proper number of the list heads in the table in order to get a better performance. The LSB of the 'key' is used to calculate the index of the head in the list heads array. This implementation is not multi-threads safe right now. Signed-off-by: Bing Zhao <bingz@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2019-11-06 13:07:56 +00:00
idx = rte_hash_crc_8byte(entry->key, 0) & h->mask;
first = &h->heads[idx];
/* No need to reuse the lookup function. */
LIST_FOREACH(node, first, next) {
if (node->key == entry->key)
return -EEXIST;
}
LIST_INSERT_HEAD(first, entry, next);
return 0;
}
void
mlx5_hlist_remove(struct mlx5_hlist *h __rte_unused,
struct mlx5_hlist_entry *entry)
{
common/mlx5: improve assert control Use the MLX5_ASSERT macros instead of the standard assert clause. Depends on the RTE_LIBRTE_MLX5_DEBUG configuration option to define it. If RTE_LIBRTE_MLX5_DEBUG is enabled MLX5_ASSERT is equal to RTE_VERIFY to bypass the global CONFIG_RTE_ENABLE_ASSERT option. If RTE_LIBRTE_MLX5_DEBUG is disabled, the global CONFIG_RTE_ENABLE_ASSERT can still make this assert active by calling RTE_VERIFY inside RTE_ASSERT. Signed-off-by: Alexander Kozyrev <akozyrev@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-01-30 16:14:40 +00:00
MLX5_ASSERT(entry && entry->next.le_prev);
net/mlx5: introduce hash list Introduce simple hash list to the mlx5 utilities. User can define its own data structure containing the mlx5_hlist_entry and create the hash list table via the creation interface. Then the entry will be inserted into the table and linked to the corresponding list head. User should guarantee there is no collision of the key and provide a callback function to handle all the remaining entries in the table when destroying the hash list. User should define a proper number of the list heads in the table in order to get a better performance. The LSB of the 'key' is used to calculate the index of the head in the list heads array. This implementation is not multi-threads safe right now. Signed-off-by: Bing Zhao <bingz@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2019-11-06 13:07:56 +00:00
LIST_REMOVE(entry, next);
/* Set to NULL to get rid of removing action for more than once. */
entry->next.le_prev = NULL;
}
void
mlx5_hlist_destroy(struct mlx5_hlist *h,
mlx5_hlist_destroy_callback_fn cb, void *ctx)
{
uint32_t idx;
struct mlx5_hlist_entry *entry;
common/mlx5: improve assert control Use the MLX5_ASSERT macros instead of the standard assert clause. Depends on the RTE_LIBRTE_MLX5_DEBUG configuration option to define it. If RTE_LIBRTE_MLX5_DEBUG is enabled MLX5_ASSERT is equal to RTE_VERIFY to bypass the global CONFIG_RTE_ENABLE_ASSERT option. If RTE_LIBRTE_MLX5_DEBUG is disabled, the global CONFIG_RTE_ENABLE_ASSERT can still make this assert active by calling RTE_VERIFY inside RTE_ASSERT. Signed-off-by: Alexander Kozyrev <akozyrev@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-01-30 16:14:40 +00:00
MLX5_ASSERT(h);
net/mlx5: introduce hash list Introduce simple hash list to the mlx5 utilities. User can define its own data structure containing the mlx5_hlist_entry and create the hash list table via the creation interface. Then the entry will be inserted into the table and linked to the corresponding list head. User should guarantee there is no collision of the key and provide a callback function to handle all the remaining entries in the table when destroying the hash list. User should define a proper number of the list heads in the table in order to get a better performance. The LSB of the 'key' is used to calculate the index of the head in the list heads array. This implementation is not multi-threads safe right now. Signed-off-by: Bing Zhao <bingz@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2019-11-06 13:07:56 +00:00
for (idx = 0; idx < h->table_sz; ++idx) {
/* no LIST_FOREACH_SAFE, using while instead */
while (!LIST_EMPTY(&h->heads[idx])) {
entry = LIST_FIRST(&h->heads[idx]);
LIST_REMOVE(entry, next);
/*
* The owner of whole element which contains data entry
* is the user, so it's the user's duty to do the clean
* up and the free work because someone may not put the
* hlist entry at the beginning(suggested to locate at
* the beginning). Or else the default free function
* will be used.
*/
if (cb)
cb(entry, ctx);
else
rte_free(entry);
}
}
rte_free(h);
}
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
static inline void
mlx5_ipool_lock(struct mlx5_indexed_pool *pool)
{
if (pool->cfg.need_lock)
rte_spinlock_lock(&pool->lock);
}
static inline void
mlx5_ipool_unlock(struct mlx5_indexed_pool *pool)
{
if (pool->cfg.need_lock)
rte_spinlock_unlock(&pool->lock);
}
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
static inline uint32_t
mlx5_trunk_idx_get(struct mlx5_indexed_pool *pool, uint32_t entry_idx)
{
struct mlx5_indexed_pool_config *cfg = &pool->cfg;
uint32_t trunk_idx = 0;
uint32_t i;
if (!cfg->grow_trunk)
return entry_idx / cfg->trunk_size;
if (entry_idx >= pool->grow_tbl[cfg->grow_trunk - 1]) {
trunk_idx = (entry_idx - pool->grow_tbl[cfg->grow_trunk - 1]) /
(cfg->trunk_size << (cfg->grow_shift *
cfg->grow_trunk)) + cfg->grow_trunk;
} else {
for (i = 0; i < cfg->grow_trunk; i++) {
if (entry_idx < pool->grow_tbl[i])
break;
}
trunk_idx = i;
}
return trunk_idx;
}
static inline uint32_t
mlx5_trunk_size_get(struct mlx5_indexed_pool *pool, uint32_t trunk_idx)
{
struct mlx5_indexed_pool_config *cfg = &pool->cfg;
return cfg->trunk_size << (cfg->grow_shift *
(trunk_idx > cfg->grow_trunk ? cfg->grow_trunk : trunk_idx));
}
static inline uint32_t
mlx5_trunk_idx_offset_get(struct mlx5_indexed_pool *pool, uint32_t trunk_idx)
{
struct mlx5_indexed_pool_config *cfg = &pool->cfg;
uint32_t offset = 0;
if (!trunk_idx)
return 0;
if (!cfg->grow_trunk)
return cfg->trunk_size * trunk_idx;
if (trunk_idx < cfg->grow_trunk)
offset = pool->grow_tbl[trunk_idx - 1];
else
offset = pool->grow_tbl[cfg->grow_trunk - 1] +
(cfg->trunk_size << (cfg->grow_shift *
cfg->grow_trunk)) * (trunk_idx - cfg->grow_trunk);
return offset;
}
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
struct mlx5_indexed_pool *
mlx5_ipool_create(struct mlx5_indexed_pool_config *cfg)
{
struct mlx5_indexed_pool *pool;
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
uint32_t i;
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
if (!cfg || !cfg->size || (!cfg->malloc ^ !cfg->free) ||
(cfg->trunk_size && ((cfg->trunk_size & (cfg->trunk_size - 1)) ||
((__builtin_ffs(cfg->trunk_size) + TRUNK_IDX_BITS) > 32))))
return NULL;
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
pool = rte_zmalloc("mlx5_ipool", sizeof(*pool) + cfg->grow_trunk *
sizeof(pool->grow_tbl[0]), RTE_CACHE_LINE_SIZE);
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
if (!pool)
return NULL;
pool->cfg = *cfg;
if (!pool->cfg.trunk_size)
pool->cfg.trunk_size = MLX5_IPOOL_DEFAULT_TRUNK_SIZE;
if (!cfg->malloc && !cfg->free) {
pool->cfg.malloc = rte_malloc_socket;
pool->cfg.free = rte_free;
}
pool->free_list = TRUNK_INVALID;
if (pool->cfg.need_lock)
rte_spinlock_init(&pool->lock);
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
/*
* Initialize the dynamic grow trunk size lookup table to have a quick
* lookup for the trunk entry index offset.
*/
for (i = 0; i < cfg->grow_trunk; i++) {
pool->grow_tbl[i] = cfg->trunk_size << (cfg->grow_shift * i);
if (i > 0)
pool->grow_tbl[i] += pool->grow_tbl[i - 1];
}
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
return pool;
}
static int
mlx5_ipool_grow(struct mlx5_indexed_pool *pool)
{
struct mlx5_indexed_trunk *trunk;
struct mlx5_indexed_trunk **trunk_tmp;
struct mlx5_indexed_trunk **p;
size_t trunk_size = 0;
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
size_t data_size;
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
size_t bmp_size;
uint32_t idx;
if (pool->n_trunk_valid == TRUNK_MAX_IDX)
return -ENOMEM;
if (pool->n_trunk_valid == pool->n_trunk) {
/* No free trunk flags, expand trunk list. */
int n_grow = pool->n_trunk_valid ? pool->n_trunk :
RTE_CACHE_LINE_SIZE / sizeof(void *);
p = pool->cfg.malloc(pool->cfg.type,
(pool->n_trunk_valid + n_grow) *
sizeof(struct mlx5_indexed_trunk *),
RTE_CACHE_LINE_SIZE, rte_socket_id());
if (!p)
return -ENOMEM;
if (pool->trunks)
memcpy(p, pool->trunks, pool->n_trunk_valid *
sizeof(struct mlx5_indexed_trunk *));
memset(RTE_PTR_ADD(p, pool->n_trunk_valid * sizeof(void *)), 0,
n_grow * sizeof(void *));
trunk_tmp = pool->trunks;
pool->trunks = p;
if (trunk_tmp)
pool->cfg.free(trunk_tmp);
pool->n_trunk += n_grow;
}
net/mlx5: add trunk release for indexed pool While entries are fully freed in trunk, it means the trunk is free now. User may prefer the free trunk memory can be reclaimed. Add the trunk release memory option for indexed pool in this case. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:01 +00:00
if (!pool->cfg.release_mem_en) {
idx = pool->n_trunk_valid;
} else {
/* Find the first available slot in trunk list */
for (idx = 0; idx < pool->n_trunk; idx++)
if (pool->trunks[idx] == NULL)
break;
}
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
trunk_size += sizeof(*trunk);
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
data_size = mlx5_trunk_size_get(pool, idx);
bmp_size = rte_bitmap_get_memory_footprint(data_size);
net/mlx5: fix indexed pool bitmap initialization Currently, the indexed memory pool bitmap start address is not aligned to cacheline size explicitly. The bitmap initialization requires the address should be cacheline aligned. In that case, the initialization maybe failed if the address is not cacheline aligned. Add RTE_CACHE_LINE_ROUNDUP() to the trunk size calculation to make sure the bitmap offset address will start with cacheline aligned. Fixes: a3cf59f56c47 ("net/mlx5: add indexed memory pool") Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Tested-by: Lijian Zhang <lijian.zhang@arm.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com> Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
2020-04-28 09:13:37 +00:00
/* rte_bitmap requires memory cacheline aligned. */
trunk_size += RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size);
trunk_size += bmp_size;
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
trunk = pool->cfg.malloc(pool->cfg.type, trunk_size,
RTE_CACHE_LINE_SIZE, rte_socket_id());
if (!trunk)
return -ENOMEM;
pool->trunks[idx] = trunk;
trunk->idx = idx;
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
trunk->free = data_size;
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
trunk->prev = TRUNK_INVALID;
trunk->next = TRUNK_INVALID;
MLX5_ASSERT(pool->free_list == TRUNK_INVALID);
pool->free_list = idx;
/* Mark all entries as available. */
net/mlx5: fix indexed pool bitmap initialization Currently, the indexed memory pool bitmap start address is not aligned to cacheline size explicitly. The bitmap initialization requires the address should be cacheline aligned. In that case, the initialization maybe failed if the address is not cacheline aligned. Add RTE_CACHE_LINE_ROUNDUP() to the trunk size calculation to make sure the bitmap offset address will start with cacheline aligned. Fixes: a3cf59f56c47 ("net/mlx5: add indexed memory pool") Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Tested-by: Lijian Zhang <lijian.zhang@arm.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com> Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
2020-04-28 09:13:37 +00:00
trunk->bmp = rte_bitmap_init_with_all_set(data_size, &trunk->data
[RTE_CACHE_LINE_ROUNDUP(data_size * pool->cfg.size)],
bmp_size);
MLX5_ASSERT(trunk->bmp);
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
pool->n_trunk_valid++;
#ifdef POOL_DEBUG
pool->trunk_new++;
pool->trunk_avail++;
#endif
return 0;
}
void *
mlx5_ipool_malloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
{
struct mlx5_indexed_trunk *trunk;
uint64_t slab = 0;
uint32_t iidx = 0;
void *p;
mlx5_ipool_lock(pool);
if (pool->free_list == TRUNK_INVALID) {
/* If no available trunks, grow new. */
if (mlx5_ipool_grow(pool)) {
mlx5_ipool_unlock(pool);
return NULL;
}
}
MLX5_ASSERT(pool->free_list != TRUNK_INVALID);
trunk = pool->trunks[pool->free_list];
MLX5_ASSERT(trunk->free);
if (!rte_bitmap_scan(trunk->bmp, &iidx, &slab)) {
mlx5_ipool_unlock(pool);
return NULL;
}
MLX5_ASSERT(slab);
iidx += __builtin_ctzll(slab);
MLX5_ASSERT(iidx != UINT32_MAX);
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
MLX5_ASSERT(iidx < mlx5_trunk_size_get(pool, trunk->idx));
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
rte_bitmap_clear(trunk->bmp, iidx);
p = &trunk->data[iidx * pool->cfg.size];
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
iidx += mlx5_trunk_idx_offset_get(pool, trunk->idx);
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
iidx += 1; /* non-zero index. */
trunk->free--;
#ifdef POOL_DEBUG
pool->n_entry++;
#endif
if (!trunk->free) {
/* Full trunk will be removed from free list in imalloc. */
MLX5_ASSERT(pool->free_list == trunk->idx);
pool->free_list = trunk->next;
if (trunk->next != TRUNK_INVALID)
pool->trunks[trunk->next]->prev = TRUNK_INVALID;
trunk->prev = TRUNK_INVALID;
trunk->next = TRUNK_INVALID;
#ifdef POOL_DEBUG
pool->trunk_empty++;
pool->trunk_avail--;
#endif
}
*idx = iidx;
mlx5_ipool_unlock(pool);
return p;
}
void *
mlx5_ipool_zmalloc(struct mlx5_indexed_pool *pool, uint32_t *idx)
{
void *entry = mlx5_ipool_malloc(pool, idx);
if (entry)
memset(entry, 0, pool->cfg.size);
return entry;
}
void
mlx5_ipool_free(struct mlx5_indexed_pool *pool, uint32_t idx)
{
struct mlx5_indexed_trunk *trunk;
uint32_t trunk_idx;
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
uint32_t entry_idx;
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
if (!idx)
return;
idx -= 1;
mlx5_ipool_lock(pool);
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
trunk_idx = mlx5_trunk_idx_get(pool, idx);
net/mlx5: add trunk release for indexed pool While entries are fully freed in trunk, it means the trunk is free now. User may prefer the free trunk memory can be reclaimed. Add the trunk release memory option for indexed pool in this case. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:01 +00:00
if ((!pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk_valid) ||
(pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk))
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
goto out;
trunk = pool->trunks[trunk_idx];
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
if (!trunk)
goto out;
entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx);
if (trunk_idx != trunk->idx ||
rte_bitmap_get(trunk->bmp, entry_idx))
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
goto out;
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
rte_bitmap_set(trunk->bmp, entry_idx);
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
trunk->free++;
net/mlx5: add trunk release for indexed pool While entries are fully freed in trunk, it means the trunk is free now. User may prefer the free trunk memory can be reclaimed. Add the trunk release memory option for indexed pool in this case. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:01 +00:00
if (pool->cfg.release_mem_en && trunk->free == mlx5_trunk_size_get
(pool, trunk->idx)) {
if (pool->free_list == trunk->idx)
pool->free_list = trunk->next;
if (trunk->next != TRUNK_INVALID)
pool->trunks[trunk->next]->prev = trunk->prev;
if (trunk->prev != TRUNK_INVALID)
pool->trunks[trunk->prev]->next = trunk->next;
pool->cfg.free(trunk);
pool->trunks[trunk_idx] = NULL;
pool->n_trunk_valid--;
#ifdef POOL_DEBUG
pool->trunk_avail--;
pool->trunk_free++;
#endif
if (pool->n_trunk_valid == 0) {
pool->cfg.free(pool->trunks);
pool->trunks = NULL;
pool->n_trunk = 0;
}
} else if (trunk->free == 1) {
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
/* Put into free trunk list head. */
MLX5_ASSERT(pool->free_list != trunk->idx);
trunk->next = pool->free_list;
trunk->prev = TRUNK_INVALID;
if (pool->free_list != TRUNK_INVALID)
pool->trunks[pool->free_list]->prev = trunk->idx;
pool->free_list = trunk->idx;
#ifdef POOL_DEBUG
pool->trunk_empty--;
pool->trunk_avail++;
#endif
}
#ifdef POOL_DEBUG
pool->n_entry--;
#endif
out:
mlx5_ipool_unlock(pool);
}
void *
mlx5_ipool_get(struct mlx5_indexed_pool *pool, uint32_t idx)
{
struct mlx5_indexed_trunk *trunk;
void *p = NULL;
uint32_t trunk_idx;
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
uint32_t entry_idx;
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
if (!idx)
return NULL;
idx -= 1;
mlx5_ipool_lock(pool);
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
trunk_idx = mlx5_trunk_idx_get(pool, idx);
net/mlx5: add trunk release for indexed pool While entries are fully freed in trunk, it means the trunk is free now. User may prefer the free trunk memory can be reclaimed. Add the trunk release memory option for indexed pool in this case. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:01 +00:00
if ((!pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk_valid) ||
(pool->cfg.release_mem_en && trunk_idx >= pool->n_trunk))
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
goto out;
trunk = pool->trunks[trunk_idx];
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
if (!trunk)
goto out;
entry_idx = idx - mlx5_trunk_idx_offset_get(pool, trunk->idx);
if (trunk_idx != trunk->idx ||
rte_bitmap_get(trunk->bmp, entry_idx))
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
goto out;
net/mlx5: add trunk dynamic grow for indexed pool This commit add trunk dynamic grow for the indexed pool. In case for pools which are not sure the entry number needed, pools can be configured in increase progressively mode. It means the trunk size will be increased dynamically one after one, then reach a stable value. It saves memory to avoid allocate a very big trunk at beginning. User should set both the grow_shift and grow_trunk to help the trunk grow works. Keep one or both grow_shift and grow_trunk as 0 makes the trunk work as fixed size. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:42:00 +00:00
p = &trunk->data[entry_idx * pool->cfg.size];
net/mlx5: add indexed memory pool Currently, the memory allocated by rte_malloc() also introduced more than 64 bytes overhead. It means when allocate 64 bytes memory, the real cost in memory maybe double. And the libc malloc() overhead is 16 bytes, If users try allocating millions of small memory blocks, the overhead costing maybe huge. And save the memory pointer will also be quite expensive. Indexed memory pool is introduced to save the memory for allocating huge amount of small memory blocks. The indexed memory uses trunk and bitmap to manage the memory entries. While the pool is empty, the trunk slot contains memory entry array will be allocated firstly. The bitmap in the trunk records the entry allocation. The offset of trunk slot in the pool and the offset of memory entry in the trunk slot compose the index for the memory entry. So, by the index, it will be very easy to address the memory of the entry. User saves the 32 bits index for the memory resource instead of the 64 bits pointer. User should create different pools for allocating different size of small memory block. It means one pool provides one fixed size of small memory blocked allocating. Signed-off-by: Suanming Mou <suanmingm@mellanox.com> Acked-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
2020-04-16 02:41:59 +00:00
out:
mlx5_ipool_unlock(pool);
return p;
}
int
mlx5_ipool_destroy(struct mlx5_indexed_pool *pool)
{
struct mlx5_indexed_trunk **trunks;
uint32_t i;
MLX5_ASSERT(pool);
mlx5_ipool_lock(pool);
trunks = pool->trunks;
for (i = 0; i < pool->n_trunk; i++) {
if (trunks[i])
pool->cfg.free(trunks[i]);
}
if (!pool->trunks)
pool->cfg.free(pool->trunks);
mlx5_ipool_unlock(pool);
rte_free(pool);
return 0;
}
void
mlx5_ipool_dump(struct mlx5_indexed_pool *pool)
{
printf("Pool %s entry size %u, trunks %u, %d entry per trunk, "
"total: %d\n",
pool->cfg.type, pool->cfg.size, pool->n_trunk_valid,
pool->cfg.trunk_size, pool->n_trunk_valid);
#ifdef POOL_DEBUG
printf("Pool %s entry %u, trunk alloc %u, empty: %u, "
"available %u free %u\n",
pool->cfg.type, pool->n_entry, pool->trunk_new,
pool->trunk_empty, pool->trunk_avail, pool->trunk_free);
#endif
}
Reference in New Issue Copy Permalink