diff --git a/drivers/common/mlx5/linux/mlx5_common_os.c b/drivers/common/mlx5/linux/mlx5_common_os.c index 5cf9576921..fba8245b8b 100644 --- a/drivers/common/mlx5/linux/mlx5_common_os.c +++ b/drivers/common/mlx5/linux/mlx5_common_os.c @@ -15,7 +15,7 @@ #include <rte_string_fns.h> #include "mlx5_common.h" -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_glue.h" #ifdef MLX5_GLUE diff --git a/drivers/common/mlx5/linux/mlx5_nl.c b/drivers/common/mlx5/linux/mlx5_nl.c index 752c57b33d..f0d04f9473 100644 --- a/drivers/common/mlx5/linux/mlx5_nl.c +++ b/drivers/common/mlx5/linux/mlx5_nl.c @@ -20,7 +20,7 @@ #include <rte_errno.h> #include "mlx5_nl.h" -#include "mlx5_common_utils.h" +#include "../mlx5_common_log.h" #include "mlx5_malloc.h" #ifdef HAVE_DEVLINK #include <linux/devlink.h> diff --git a/drivers/common/mlx5/meson.build b/drivers/common/mlx5/meson.build index e78b4f47bc..fa2b8b9834 100644 --- a/drivers/common/mlx5/meson.build +++ b/drivers/common/mlx5/meson.build @@ -16,6 +16,7 @@ sources += files( 'mlx5_malloc.c', 'mlx5_common_pci.c', 'mlx5_common_devx.c', + 'mlx5_common_utils.c', ) cflags_options = [ diff --git a/drivers/common/mlx5/mlx5_common.c b/drivers/common/mlx5/mlx5_common.c index f92f05bda5..d397459a3d 100644 --- a/drivers/common/mlx5/mlx5_common.c +++ b/drivers/common/mlx5/mlx5_common.c @@ -11,7 +11,7 @@ #include "mlx5_common.h" #include "mlx5_common_os.h" -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_common_pci.h" uint8_t haswell_broadwell_cpu; diff --git a/drivers/common/mlx5/mlx5_common_devx.c b/drivers/common/mlx5/mlx5_common_devx.c index d19be122bd..22c8d356c4 100644 --- a/drivers/common/mlx5/mlx5_common_devx.c +++ b/drivers/common/mlx5/mlx5_common_devx.c @@ -12,7 +12,7 @@ #include "mlx5_prm.h" #include "mlx5_devx_cmds.h" -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_malloc.h" #include "mlx5_common.h" #include "mlx5_common_devx.h" diff --git a/drivers/common/mlx5/mlx5_common_log.h b/drivers/common/mlx5/mlx5_common_log.h new file mode 100644 index 0000000000..26b13fedaf --- /dev/null +++ b/drivers/common/mlx5/mlx5_common_log.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2019 Mellanox Technologies, Ltd + */ + +#ifndef RTE_PMD_MLX5_COMMON_LOG_H_ +#define RTE_PMD_MLX5_COMMON_LOG_H_ + +#include "mlx5_common.h" + + +extern int mlx5_common_logtype; + +#define MLX5_COMMON_LOG_PREFIX "mlx5_common" +/* Generic printf()-like logging macro with automatic line feed. */ +#define DRV_LOG(level, ...) \ + PMD_DRV_LOG_(level, mlx5_common_logtype, MLX5_COMMON_LOG_PREFIX, \ + __VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \ + PMD_DRV_LOG_CPAREN) + +#endif /* RTE_PMD_MLX5_COMMON_LOG_H_ */ + diff --git a/drivers/common/mlx5/mlx5_common_mp.c b/drivers/common/mlx5/mlx5_common_mp.c index 40e3956e45..673a7c31de 100644 --- a/drivers/common/mlx5/mlx5_common_mp.c +++ b/drivers/common/mlx5/mlx5_common_mp.c @@ -10,7 +10,7 @@ #include <rte_errno.h> #include "mlx5_common_mp.h" -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_malloc.h" /** diff --git a/drivers/common/mlx5/mlx5_common_mr.c b/drivers/common/mlx5/mlx5_common_mr.c index e1ed0caf3a..afb5b3d0a7 100644 --- a/drivers/common/mlx5/mlx5_common_mr.c +++ b/drivers/common/mlx5/mlx5_common_mr.c @@ -11,7 +11,7 @@ #include "mlx5_glue.h" #include "mlx5_common_mp.h" #include "mlx5_common_mr.h" -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_malloc.h" struct mr_find_contig_memsegs_data { diff --git a/drivers/common/mlx5/mlx5_common_pci.c b/drivers/common/mlx5/mlx5_common_pci.c index 5a73ffa60a..3f16cd21cf 100644 --- a/drivers/common/mlx5/mlx5_common_pci.c +++ b/drivers/common/mlx5/mlx5_common_pci.c @@ -4,7 +4,9 @@ #include <stdlib.h> #include <rte_malloc.h> -#include "mlx5_common_utils.h" +#include <rte_class.h> + +#include "mlx5_common_log.h" #include "mlx5_common_pci.h" struct mlx5_pci_device { diff --git a/drivers/common/mlx5/mlx5_common_utils.c b/drivers/common/mlx5/mlx5_common_utils.c new file mode 100644 index 0000000000..ad2011e858 --- /dev/null +++ b/drivers/common/mlx5/mlx5_common_utils.c @@ -0,0 +1,221 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2019 Mellanox Technologies, Ltd + */ + +#include <rte_malloc.h> +#include <rte_hash_crc.h> +#include <rte_errno.h> + +#include <mlx5_malloc.h> + +#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" + +/********************* Hash List **********************/ + +static struct mlx5_hlist_entry * +mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused, + void *ctx __rte_unused) +{ + return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY); +} + +static void +mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused, + struct mlx5_hlist_entry *entry) +{ + mlx5_free(entry); +} + +struct mlx5_hlist * +mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size, + uint32_t flags, mlx5_hlist_create_cb cb_create, + mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove) +{ + struct mlx5_hlist *h; + uint32_t act_size; + uint32_t alloc_size; + uint32_t i; + + if (!size || !cb_match || (!cb_create ^ !cb_remove)) + return NULL; + /* Align to the next power of 2, 32bits integer is enough now. */ + if (!rte_is_power_of_2(size)) { + act_size = rte_align32pow2(size); + DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, " + "will be aligned to 0x%" PRIX32 ".", size, act_size); + } else { + act_size = size; + } + alloc_size = sizeof(struct mlx5_hlist) + + sizeof(struct mlx5_hlist_bucket) * act_size; + /* Using zmalloc, then no need to initialize the heads. */ + h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE, + SOCKET_ID_ANY); + if (!h) { + DRV_LOG(ERR, "No memory for hash list %s creation", + name ? name : "None"); + return NULL; + } + if (name) + snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name); + h->table_sz = act_size; + h->mask = act_size - 1; + h->entry_sz = entry_size; + h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY); + h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST); + h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb; + h->cb_match = cb_match; + h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb; + for (i = 0; i < act_size; i++) + rte_rwlock_init(&h->buckets[i].lock); + DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.", + h->name, act_size); + return h; +} + +static struct mlx5_hlist_entry * +__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx, + void *ctx, bool reuse) +{ + struct mlx5_hlist_head *first; + struct mlx5_hlist_entry *node; + + MLX5_ASSERT(h); + first = &h->buckets[idx].head; + LIST_FOREACH(node, first, next) { + if (!h->cb_match(h, node, key, ctx)) { + if (reuse) { + __atomic_add_fetch(&node->ref_cnt, 1, + __ATOMIC_RELAXED); + DRV_LOG(DEBUG, "Hash list %s entry %p " + "reuse: %u.", + h->name, (void *)node, node->ref_cnt); + } + break; + } + } + return node; +} + +static struct mlx5_hlist_entry * +hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx, + void *ctx, bool reuse) +{ + struct mlx5_hlist_entry *node; + + MLX5_ASSERT(h); + rte_rwlock_read_lock(&h->buckets[idx].lock); + node = __hlist_lookup(h, key, idx, ctx, reuse); + rte_rwlock_read_unlock(&h->buckets[idx].lock); + return node; +} + +struct mlx5_hlist_entry * +mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx) +{ + uint32_t idx; + + if (h->direct_key) + idx = (uint32_t)(key & h->mask); + else + idx = rte_hash_crc_8byte(key, 0) & h->mask; + return hlist_lookup(h, key, idx, ctx, false); +} + +struct mlx5_hlist_entry* +mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx) +{ + uint32_t idx; + struct mlx5_hlist_head *first; + struct mlx5_hlist_bucket *b; + struct mlx5_hlist_entry *entry; + uint32_t prev_gen_cnt = 0; + + if (h->direct_key) + idx = (uint32_t)(key & h->mask); + else + idx = rte_hash_crc_8byte(key, 0) & h->mask; + MLX5_ASSERT(h); + b = &h->buckets[idx]; + /* Use write lock directly for write-most list. */ + if (!h->write_most) { + prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE); + entry = hlist_lookup(h, key, idx, ctx, true); + if (entry) + return entry; + } + rte_rwlock_write_lock(&b->lock); + /* Check if the list changed by other threads. */ + if (h->write_most || + prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) { + entry = __hlist_lookup(h, key, idx, ctx, true); + if (entry) + goto done; + } + first = &b->head; + entry = h->cb_create(h, key, ctx); + if (!entry) { + rte_errno = ENOMEM; + DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name); + goto done; + } + entry->idx = idx; + entry->ref_cnt = 1; + LIST_INSERT_HEAD(first, entry, next); + __atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL); + DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.", + h->name, (void *)entry, entry->ref_cnt); +done: + rte_rwlock_write_unlock(&b->lock); + return entry; +} + +int +mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry) +{ + uint32_t idx = entry->idx; + + rte_rwlock_write_lock(&h->buckets[idx].lock); + MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev); + DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.", + h->name, (void *)entry, entry->ref_cnt); + if (--entry->ref_cnt) { + rte_rwlock_write_unlock(&h->buckets[idx].lock); + return 1; + } + LIST_REMOVE(entry, next); + /* Set to NULL to get rid of removing action for more than once. */ + entry->next.le_prev = NULL; + h->cb_remove(h, entry); + rte_rwlock_write_unlock(&h->buckets[idx].lock); + DRV_LOG(DEBUG, "Hash list %s entry %p removed.", + h->name, (void *)entry); + return 0; +} + +void +mlx5_hlist_destroy(struct mlx5_hlist *h) +{ + uint32_t idx; + struct mlx5_hlist_entry *entry; + + MLX5_ASSERT(h); + for (idx = 0; idx < h->table_sz; ++idx) { + /* No LIST_FOREACH_SAFE, using while instead. */ + while (!LIST_EMPTY(&h->buckets[idx].head)) { + entry = LIST_FIRST(&h->buckets[idx].head); + LIST_REMOVE(entry, next); + /* + * The owner of whole element which contains data entry + * is the user, so it's the user's duty to do the clean + * up and the free work because someone may not put the + * hlist entry at the beginning(suggested to locate at + * the beginning). Or else the default free function + * will be used. + */ + h->cb_remove(h, entry); + } + } + mlx5_free(h); +} diff --git a/drivers/common/mlx5/mlx5_common_utils.h b/drivers/common/mlx5/mlx5_common_utils.h index 6cba39c8cc..ed378ce9bd 100644 --- a/drivers/common/mlx5/mlx5_common_utils.h +++ b/drivers/common/mlx5/mlx5_common_utils.h @@ -7,14 +7,202 @@ #include "mlx5_common.h" +#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */ +#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */ -extern int mlx5_common_logtype; +/** Maximum size of string for naming the hlist table. */ +#define MLX5_HLIST_NAMESIZE 32 -#define MLX5_COMMON_LOG_PREFIX "mlx5_common" -/* Generic printf()-like logging macro with automatic line feed. */ -#define DRV_LOG(level, ...) \ - PMD_DRV_LOG_(level, mlx5_common_logtype, MLX5_COMMON_LOG_PREFIX, \ - __VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \ - PMD_DRV_LOG_CPAREN) +struct mlx5_hlist; + +/** + * Structure of the entry in the hash list, user should define its own struct + * that contains this in order to store the data. The 'key' is 64-bits right + * now and its user's responsibility to guarantee there is no collision. + */ +struct mlx5_hlist_entry { + LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */ + uint32_t idx; /* Bucket index the entry belongs to. */ + uint32_t ref_cnt; /* Reference count. */ +}; + +/** Structure for hash head. */ +LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry); + +/** + * Type of callback function for entry removal. + * + * @param list + * The hash list. + * @param entry + * The entry in the list. + */ +typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry); + +/** + * Type of function for user defined matching. + * + * @param list + * The hash list. + * @param entry + * The entry in the list. + * @param key + * The new entry key. + * @param ctx + * The pointer to new entry context. + * + * @return + * 0 if matching, non-zero number otherwise. + */ +typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list, + struct mlx5_hlist_entry *entry, + uint64_t key, void *ctx); + +/** + * Type of function for user defined hash list entry creation. + * + * @param list + * The hash list. + * @param key + * The key of the new entry. + * @param ctx + * The pointer to new entry context. + * + * @return + * Pointer to allocated entry on success, NULL otherwise. + */ +typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb) + (struct mlx5_hlist *list, + uint64_t key, void *ctx); + +/* Hash list bucket head. */ +struct mlx5_hlist_bucket { + struct mlx5_hlist_head head; /* List head. */ + rte_rwlock_t lock; /* Bucket lock. */ + uint32_t gen_cnt; /* List modification will update generation count. */ +} __rte_cache_aligned; + +/** + * Hash list table structure + * + * Entry in hash list could be reused if entry already exists, reference + * count will increase and the existing entry returns. + * + * When destroy an entry from list, decrease reference count and only + * destroy when no further reference. + */ +struct mlx5_hlist { + char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */ + /**< number of heads, need to be power of 2. */ + uint32_t table_sz; + uint32_t entry_sz; /**< Size of entry, used to allocate entry. */ + /**< mask to get the index of the list heads. */ + uint32_t mask; + bool direct_key; /* Use the new entry key directly as hash index. */ + bool write_most; /* List mostly used for append new or destroy. */ + void *ctx; + mlx5_hlist_create_cb cb_create; /**< entry create callback. */ + mlx5_hlist_match_cb cb_match; /**< entry match callback. */ + mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */ + struct mlx5_hlist_bucket buckets[] __rte_cache_aligned; + /**< list bucket arrays. */ +}; + +/** + * Create a hash list table, the user can specify the list heads array size + * of the table, now the size should be a power of 2 in order to get better + * distribution for the entries. Each entry is a part of the whole data element + * and the caller should be responsible for the data element's allocation and + * cleanup / free. Key of each entry will be calculated with CRC in order to + * generate a little fairer distribution. + * + * @param name + * Name of the hash list(optional). + * @param size + * Heads array size of the hash list. + * @param entry_size + * Entry size to allocate if cb_create not specified. + * @param flags + * The hash list attribute flags. + * @param cb_create + * Callback function for entry create. + * @param cb_match + * Callback function for entry match. + * @param cb_destroy + * Callback function for entry destroy. + * @return + * Pointer of the hash list table created, NULL on failure. + */ +__rte_internal +struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size, + uint32_t entry_size, uint32_t flags, + mlx5_hlist_create_cb cb_create, + mlx5_hlist_match_cb cb_match, + mlx5_hlist_remove_cb cb_destroy); + +/** + * Search an entry matching the key. + * + * Result returned might be destroyed by other thread, must use + * this function only in main thread. + * + * @param h + * Pointer to the hast list table. + * @param key + * Key for the searching entry. + * @param ctx + * Common context parameter used by entry callback function. + * + * @return + * Pointer of the hlist entry if found, NULL otherwise. + */ +__rte_internal +struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, + void *ctx); + +/** + * Insert an entry to the hash list table, the entry is only part of whole data + * element and a 64B key is used for matching. User should construct the key or + * give a calculated hash signature and guarantee there is no collision. + * + * @param h + * Pointer to the hast list table. + * @param entry + * Entry to be inserted into the hash list table. + * @param ctx + * Common context parameter used by callback function. + * + * @return + * registered entry on success, NULL otherwise + */ +__rte_internal +struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, + void *ctx); + +/** + * Remove an entry from the hash list table. User should guarantee the validity + * of the entry. + * + * @param h + * Pointer to the hast list table. (not used) + * @param entry + * Entry to be removed from the hash list table. + * @return + * 0 on entry removed, 1 on entry still referenced. + */ +__rte_internal +int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry); + +/** + * Destroy the hash list table, all the entries already inserted into the lists + * will be handled by the callback function provided by the user (including + * free if needed) before the table is freed. + * + * @param h + * Pointer to the hast list table. + */ +__rte_internal +void mlx5_hlist_destroy(struct mlx5_hlist *h); #endif /* RTE_PMD_MLX5_COMMON_UTILS_H_ */ diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c index 703b18cc32..6b7684481f 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.c +++ b/drivers/common/mlx5/mlx5_devx_cmds.c @@ -9,7 +9,7 @@ #include "mlx5_prm.h" #include "mlx5_devx_cmds.h" -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_malloc.h" diff --git a/drivers/common/mlx5/mlx5_malloc.c b/drivers/common/mlx5/mlx5_malloc.c index 9d30cedbaa..b19501e1bc 100644 --- a/drivers/common/mlx5/mlx5_malloc.c +++ b/drivers/common/mlx5/mlx5_malloc.c @@ -8,7 +8,7 @@ #include <stdbool.h> #include <string.h> -#include "mlx5_common_utils.h" +#include "mlx5_common_log.h" #include "mlx5_common_os.h" #include "mlx5_malloc.h" diff --git a/drivers/common/mlx5/version.map b/drivers/common/mlx5/version.map index 89f0ee04cb..40b0d713ac 100644 --- a/drivers/common/mlx5/version.map +++ b/drivers/common/mlx5/version.map @@ -70,6 +70,12 @@ INTERNAL { mlx5_glue; + mlx5_hlist_create; + mlx5_hlist_lookup; + mlx5_hlist_register; + mlx5_hlist_unregister; + mlx5_hlist_destroy; + mlx5_malloc; mlx5_malloc_mem_select; diff --git a/drivers/common/mlx5/windows/mlx5_common_os.c b/drivers/common/mlx5/windows/mlx5_common_os.c index f2d781a965..2e6e172a96 100644 --- a/drivers/common/mlx5/windows/mlx5_common_os.c +++ b/drivers/common/mlx5/windows/mlx5_common_os.c @@ -11,7 +11,7 @@ #include <rte_errno.h> #include "mlx5_devx_cmds.h" -#include "mlx5_common_utils.h" +#include "../mlx5_common_log.h" #include "mlx5_common.h" #include "mlx5_common_os.h" #include "mlx5_malloc.h" diff --git a/drivers/common/mlx5/windows/mlx5_glue.c b/drivers/common/mlx5/windows/mlx5_glue.c index aef6d3b5f4..535487a8d4 100644 --- a/drivers/common/mlx5/windows/mlx5_glue.c +++ b/drivers/common/mlx5/windows/mlx5_glue.c @@ -12,7 +12,7 @@ #include <rte_malloc.h> #include "mlx5_glue.h" -#include "mlx5_common_utils.h" +#include "../mlx5_common_log.h" #include "mlx5_win_ext.h" /* diff --git a/drivers/net/mlx5/mlx5_utils.c b/drivers/net/mlx5/mlx5_utils.c index a39b5edddc..18fe23e4fb 100644 --- a/drivers/net/mlx5/mlx5_utils.c +++ b/drivers/net/mlx5/mlx5_utils.c @@ -3,220 +3,11 @@ */ #include <rte_malloc.h> -#include <rte_hash_crc.h> #include <mlx5_malloc.h> #include "mlx5_utils.h" -/********************* Hash List **********************/ - -static struct mlx5_hlist_entry * -mlx5_hlist_default_create_cb(struct mlx5_hlist *h, uint64_t key __rte_unused, - void *ctx __rte_unused) -{ - return mlx5_malloc(MLX5_MEM_ZERO, h->entry_sz, 0, SOCKET_ID_ANY); -} - -static void -mlx5_hlist_default_remove_cb(struct mlx5_hlist *h __rte_unused, - struct mlx5_hlist_entry *entry) -{ - mlx5_free(entry); -} - -struct mlx5_hlist * -mlx5_hlist_create(const char *name, uint32_t size, uint32_t entry_size, - uint32_t flags, mlx5_hlist_create_cb cb_create, - mlx5_hlist_match_cb cb_match, mlx5_hlist_remove_cb cb_remove) -{ - struct mlx5_hlist *h; - uint32_t act_size; - uint32_t alloc_size; - uint32_t i; - - if (!size || !cb_match || (!cb_create ^ !cb_remove)) - return NULL; - /* Align to the next power of 2, 32bits integer is enough now. */ - if (!rte_is_power_of_2(size)) { - act_size = rte_align32pow2(size); - DRV_LOG(DEBUG, "Size 0x%" PRIX32 " is not power of 2, " - "will be aligned to 0x%" PRIX32 ".", size, act_size); - } else { - act_size = size; - } - alloc_size = sizeof(struct mlx5_hlist) + - sizeof(struct mlx5_hlist_bucket) * act_size; - /* Using zmalloc, then no need to initialize the heads. */ - h = mlx5_malloc(MLX5_MEM_ZERO, alloc_size, RTE_CACHE_LINE_SIZE, - SOCKET_ID_ANY); - if (!h) { - DRV_LOG(ERR, "No memory for hash list %s creation", - name ? name : "None"); - return NULL; - } - if (name) - snprintf(h->name, MLX5_HLIST_NAMESIZE, "%s", name); - h->table_sz = act_size; - h->mask = act_size - 1; - h->entry_sz = entry_size; - h->direct_key = !!(flags & MLX5_HLIST_DIRECT_KEY); - h->write_most = !!(flags & MLX5_HLIST_WRITE_MOST); - h->cb_create = cb_create ? cb_create : mlx5_hlist_default_create_cb; - h->cb_match = cb_match; - h->cb_remove = cb_remove ? cb_remove : mlx5_hlist_default_remove_cb; - for (i = 0; i < act_size; i++) - rte_rwlock_init(&h->buckets[i].lock); - DRV_LOG(DEBUG, "Hash list with %s size 0x%" PRIX32 " is created.", - h->name, act_size); - return h; -} - -static struct mlx5_hlist_entry * -__hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx, - void *ctx, bool reuse) -{ - struct mlx5_hlist_head *first; - struct mlx5_hlist_entry *node; - - MLX5_ASSERT(h); - first = &h->buckets[idx].head; - LIST_FOREACH(node, first, next) { - if (!h->cb_match(h, node, key, ctx)) { - if (reuse) { - __atomic_add_fetch(&node->ref_cnt, 1, - __ATOMIC_RELAXED); - DRV_LOG(DEBUG, "Hash list %s entry %p " - "reuse: %u.", - h->name, (void *)node, node->ref_cnt); - } - break; - } - } - return node; -} - -static struct mlx5_hlist_entry * -hlist_lookup(struct mlx5_hlist *h, uint64_t key, uint32_t idx, - void *ctx, bool reuse) -{ - struct mlx5_hlist_entry *node; - - MLX5_ASSERT(h); - rte_rwlock_read_lock(&h->buckets[idx].lock); - node = __hlist_lookup(h, key, idx, ctx, reuse); - rte_rwlock_read_unlock(&h->buckets[idx].lock); - return node; -} - -struct mlx5_hlist_entry * -mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, void *ctx) -{ - uint32_t idx; - - if (h->direct_key) - idx = (uint32_t)(key & h->mask); - else - idx = rte_hash_crc_8byte(key, 0) & h->mask; - return hlist_lookup(h, key, idx, ctx, false); -} - -struct mlx5_hlist_entry* -mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, void *ctx) -{ - uint32_t idx; - struct mlx5_hlist_head *first; - struct mlx5_hlist_bucket *b; - struct mlx5_hlist_entry *entry; - uint32_t prev_gen_cnt = 0; - - if (h->direct_key) - idx = (uint32_t)(key & h->mask); - else - idx = rte_hash_crc_8byte(key, 0) & h->mask; - MLX5_ASSERT(h); - b = &h->buckets[idx]; - /* Use write lock directly for write-most list. */ - if (!h->write_most) { - prev_gen_cnt = __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE); - entry = hlist_lookup(h, key, idx, ctx, true); - if (entry) - return entry; - } - rte_rwlock_write_lock(&b->lock); - /* Check if the list changed by other threads. */ - if (h->write_most || - prev_gen_cnt != __atomic_load_n(&b->gen_cnt, __ATOMIC_ACQUIRE)) { - entry = __hlist_lookup(h, key, idx, ctx, true); - if (entry) - goto done; - } - first = &b->head; - entry = h->cb_create(h, key, ctx); - if (!entry) { - rte_errno = ENOMEM; - DRV_LOG(DEBUG, "Can't allocate hash list %s entry.", h->name); - goto done; - } - entry->idx = idx; - entry->ref_cnt = 1; - LIST_INSERT_HEAD(first, entry, next); - __atomic_add_fetch(&b->gen_cnt, 1, __ATOMIC_ACQ_REL); - DRV_LOG(DEBUG, "Hash list %s entry %p new: %u.", - h->name, (void *)entry, entry->ref_cnt); -done: - rte_rwlock_write_unlock(&b->lock); - return entry; -} - -int -mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry) -{ - uint32_t idx = entry->idx; - - rte_rwlock_write_lock(&h->buckets[idx].lock); - MLX5_ASSERT(entry && entry->ref_cnt && entry->next.le_prev); - DRV_LOG(DEBUG, "Hash list %s entry %p deref: %u.", - h->name, (void *)entry, entry->ref_cnt); - if (--entry->ref_cnt) { - rte_rwlock_write_unlock(&h->buckets[idx].lock); - return 1; - } - LIST_REMOVE(entry, next); - /* Set to NULL to get rid of removing action for more than once. */ - entry->next.le_prev = NULL; - h->cb_remove(h, entry); - rte_rwlock_write_unlock(&h->buckets[idx].lock); - DRV_LOG(DEBUG, "Hash list %s entry %p removed.", - h->name, (void *)entry); - return 0; -} - -void -mlx5_hlist_destroy(struct mlx5_hlist *h) -{ - uint32_t idx; - struct mlx5_hlist_entry *entry; - - MLX5_ASSERT(h); - for (idx = 0; idx < h->table_sz; ++idx) { - /* No LIST_FOREACH_SAFE, using while instead. */ - while (!LIST_EMPTY(&h->buckets[idx].head)) { - entry = LIST_FIRST(&h->buckets[idx].head); - LIST_REMOVE(entry, next); - /* - * The owner of whole element which contains data entry - * is the user, so it's the user's duty to do the clean - * up and the free work because someone may not put the - * hlist entry at the beginning(suggested to locate at - * the beginning). Or else the default free function - * will be used. - */ - h->cb_remove(h, entry); - } - } - mlx5_free(h); -} /********************* Cache list ************************/ diff --git a/drivers/net/mlx5/mlx5_utils.h b/drivers/net/mlx5/mlx5_utils.h index 289941cebc..b54517c6df 100644 --- a/drivers/net/mlx5/mlx5_utils.h +++ b/drivers/net/mlx5/mlx5_utils.h @@ -18,6 +18,7 @@ #include <rte_bitmap.h> #include <mlx5_common.h> +#include <mlx5_common_utils.h> #include "mlx5_defs.h" @@ -261,199 +262,6 @@ log2above(unsigned int v) return l + r; } -#define MLX5_HLIST_DIRECT_KEY 0x0001 /* Use the key directly as hash index. */ -#define MLX5_HLIST_WRITE_MOST 0x0002 /* List mostly used for append new. */ - -/** Maximum size of string for naming the hlist table. */ -#define MLX5_HLIST_NAMESIZE 32 - -struct mlx5_hlist; - -/** - * Structure of the entry in the hash list, user should define its own struct - * that contains this in order to store the data. The 'key' is 64-bits right - * now and its user's responsibility to guarantee there is no collision. - */ -struct mlx5_hlist_entry { - LIST_ENTRY(mlx5_hlist_entry) next; /* entry pointers in the list. */ - uint32_t idx; /* Bucket index the entry belongs to. */ - uint32_t ref_cnt; /* Reference count. */ -}; - -/** Structure for hash head. */ -LIST_HEAD(mlx5_hlist_head, mlx5_hlist_entry); - -/** - * Type of callback function for entry removal. - * - * @param list - * The hash list. - * @param entry - * The entry in the list. - */ -typedef void (*mlx5_hlist_remove_cb)(struct mlx5_hlist *list, - struct mlx5_hlist_entry *entry); - -/** - * Type of function for user defined matching. - * - * @param list - * The hash list. - * @param entry - * The entry in the list. - * @param key - * The new entry key. - * @param ctx - * The pointer to new entry context. - * - * @return - * 0 if matching, non-zero number otherwise. - */ -typedef int (*mlx5_hlist_match_cb)(struct mlx5_hlist *list, - struct mlx5_hlist_entry *entry, - uint64_t key, void *ctx); - -/** - * Type of function for user defined hash list entry creation. - * - * @param list - * The hash list. - * @param key - * The key of the new entry. - * @param ctx - * The pointer to new entry context. - * - * @return - * Pointer to allocated entry on success, NULL otherwise. - */ -typedef struct mlx5_hlist_entry *(*mlx5_hlist_create_cb) - (struct mlx5_hlist *list, - uint64_t key, void *ctx); - -/* Hash list bucket head. */ -struct mlx5_hlist_bucket { - struct mlx5_hlist_head head; /* List head. */ - rte_rwlock_t lock; /* Bucket lock. */ - uint32_t gen_cnt; /* List modification will update generation count. */ -} __rte_cache_aligned; - -/** - * Hash list table structure - * - * Entry in hash list could be reused if entry already exists, reference - * count will increase and the existing entry returns. - * - * When destroy an entry from list, decrease reference count and only - * destroy when no further reference. - */ -struct mlx5_hlist { - char name[MLX5_HLIST_NAMESIZE]; /**< Name of the hash list. */ - /**< number of heads, need to be power of 2. */ - uint32_t table_sz; - uint32_t entry_sz; /**< Size of entry, used to allocate entry. */ - /**< mask to get the index of the list heads. */ - uint32_t mask; - bool direct_key; /* Use the new entry key directly as hash index. */ - bool write_most; /* List mostly used for append new or destroy. */ - void *ctx; - mlx5_hlist_create_cb cb_create; /**< entry create callback. */ - mlx5_hlist_match_cb cb_match; /**< entry match callback. */ - mlx5_hlist_remove_cb cb_remove; /**< entry remove callback. */ - struct mlx5_hlist_bucket buckets[] __rte_cache_aligned; - /**< list bucket arrays. */ -}; - -/** - * Create a hash list table, the user can specify the list heads array size - * of the table, now the size should be a power of 2 in order to get better - * distribution for the entries. Each entry is a part of the whole data element - * and the caller should be responsible for the data element's allocation and - * cleanup / free. Key of each entry will be calculated with CRC in order to - * generate a little fairer distribution. - * - * @param name - * Name of the hash list(optional). - * @param size - * Heads array size of the hash list. - * @param entry_size - * Entry size to allocate if cb_create not specified. - * @param flags - * The hash list attribute flags. - * @param cb_create - * Callback function for entry create. - * @param cb_match - * Callback function for entry match. - * @param cb_destroy - * Callback function for entry destroy. - * @return - * Pointer of the hash list table created, NULL on failure. - */ -struct mlx5_hlist *mlx5_hlist_create(const char *name, uint32_t size, - uint32_t entry_size, uint32_t flags, - mlx5_hlist_create_cb cb_create, - mlx5_hlist_match_cb cb_match, - mlx5_hlist_remove_cb cb_destroy); - -/** - * Search an entry matching the key. - * - * Result returned might be destroyed by other thread, must use - * this function only in main thread. - * - * @param h - * Pointer to the hast list table. - * @param key - * Key for the searching entry. - * @param ctx - * Common context parameter used by entry callback function. - * - * @return - * Pointer of the hlist entry if found, NULL otherwise. - */ -struct mlx5_hlist_entry *mlx5_hlist_lookup(struct mlx5_hlist *h, uint64_t key, - void *ctx); - -/** - * Insert an entry to the hash list table, the entry is only part of whole data - * element and a 64B key is used for matching. User should construct the key or - * give a calculated hash signature and guarantee there is no collision. - * - * @param h - * Pointer to the hast list table. - * @param entry - * Entry to be inserted into the hash list table. - * @param ctx - * Common context parameter used by callback function. - * - * @return - * registered entry on success, NULL otherwise - */ -struct mlx5_hlist_entry *mlx5_hlist_register(struct mlx5_hlist *h, uint64_t key, - void *ctx); - -/** - * Remove an entry from the hash list table. User should guarantee the validity - * of the entry. - * - * @param h - * Pointer to the hast list table. (not used) - * @param entry - * Entry to be removed from the hash list table. - * @return - * 0 on entry removed, 1 on entry still referenced. - */ -int mlx5_hlist_unregister(struct mlx5_hlist *h, struct mlx5_hlist_entry *entry); - -/** - * Destroy the hash list table, all the entries already inserted into the lists - * will be handled by the callback function provided by the user (including - * free if needed) before the table is freed. - * - * @param h - * Pointer to the hast list table. - */ -void mlx5_hlist_destroy(struct mlx5_hlist *h); - /************************ cache list *****************************/ /** Maximum size of string for naming. */