diff --git a/app/test/test_mbuf.c b/app/test/test_mbuf.c index 9fea312c85..854bc26d8f 100644 --- a/app/test/test_mbuf.c +++ b/app/test/test_mbuf.c @@ -32,6 +32,7 @@ #include #include #include +#include #include "test.h" @@ -2411,6 +2412,142 @@ test_pktmbuf_ext_shinfo_init_helper(struct rte_mempool *pktmbuf_pool) return -1; } +static int +test_mbuf_dyn(struct rte_mempool *pktmbuf_pool) +{ + const struct rte_mbuf_dynfield dynfield = { + .name = "test-dynfield", + .size = sizeof(uint8_t), + .align = __alignof__(uint8_t), + .flags = 0, + }; + const struct rte_mbuf_dynfield dynfield2 = { + .name = "test-dynfield2", + .size = sizeof(uint16_t), + .align = __alignof__(uint16_t), + .flags = 0, + }; + const struct rte_mbuf_dynfield dynfield3 = { + .name = "test-dynfield3", + .size = sizeof(uint8_t), + .align = __alignof__(uint8_t), + .flags = 0, + }; + const struct rte_mbuf_dynfield dynfield_fail_big = { + .name = "test-dynfield-fail-big", + .size = 256, + .align = 1, + .flags = 0, + }; + const struct rte_mbuf_dynfield dynfield_fail_align = { + .name = "test-dynfield-fail-align", + .size = 1, + .align = 3, + .flags = 0, + }; + const struct rte_mbuf_dynflag dynflag = { + .name = "test-dynflag", + .flags = 0, + }; + const struct rte_mbuf_dynflag dynflag2 = { + .name = "test-dynflag2", + .flags = 0, + }; + const struct rte_mbuf_dynflag dynflag3 = { + .name = "test-dynflag3", + .flags = 0, + }; + struct rte_mbuf *m = NULL; + int offset, offset2, offset3; + int flag, flag2, flag3; + int ret; + + printf("Test mbuf dynamic fields and flags\n"); + rte_mbuf_dyn_dump(stdout); + + offset = rte_mbuf_dynfield_register(&dynfield); + if (offset == -1) + GOTO_FAIL("failed to register dynamic field, offset=%d: %s", + offset, strerror(errno)); + + ret = rte_mbuf_dynfield_register(&dynfield); + if (ret != offset) + GOTO_FAIL("failed to lookup dynamic field, ret=%d: %s", + ret, strerror(errno)); + + offset2 = rte_mbuf_dynfield_register(&dynfield2); + if (offset2 == -1 || offset2 == offset || (offset2 & 1)) + GOTO_FAIL("failed to register dynamic field 2, offset2=%d: %s", + offset2, strerror(errno)); + + offset3 = rte_mbuf_dynfield_register_offset(&dynfield3, + offsetof(struct rte_mbuf, dynfield1[1])); + if (offset3 != offsetof(struct rte_mbuf, dynfield1[1])) + GOTO_FAIL("failed to register dynamic field 3, offset=%d: %s", + offset3, strerror(errno)); + + printf("dynfield: offset=%d, offset2=%d, offset3=%d\n", + offset, offset2, offset3); + + ret = rte_mbuf_dynfield_register(&dynfield_fail_big); + if (ret != -1) + GOTO_FAIL("dynamic field creation should fail (too big)"); + + ret = rte_mbuf_dynfield_register(&dynfield_fail_align); + if (ret != -1) + GOTO_FAIL("dynamic field creation should fail (bad alignment)"); + + ret = rte_mbuf_dynfield_register_offset(&dynfield_fail_align, + offsetof(struct rte_mbuf, ol_flags)); + if (ret != -1) + GOTO_FAIL("dynamic field creation should fail (not avail)"); + + flag = rte_mbuf_dynflag_register(&dynflag); + if (flag == -1) + GOTO_FAIL("failed to register dynamic flag, flag=%d: %s", + flag, strerror(errno)); + + ret = rte_mbuf_dynflag_register(&dynflag); + if (ret != flag) + GOTO_FAIL("failed to lookup dynamic flag, ret=%d: %s", + ret, strerror(errno)); + + flag2 = rte_mbuf_dynflag_register(&dynflag2); + if (flag2 == -1 || flag2 == flag) + GOTO_FAIL("failed to register dynamic flag 2, flag2=%d: %s", + flag2, strerror(errno)); + + flag3 = rte_mbuf_dynflag_register_bitnum(&dynflag3, + rte_bsf64(PKT_LAST_FREE)); + if (flag3 != rte_bsf64(PKT_LAST_FREE)) + GOTO_FAIL("failed to register dynamic flag 3, flag2=%d: %s", + flag3, strerror(errno)); + + printf("dynflag: flag=%d, flag2=%d, flag3=%d\n", flag, flag2, flag3); + + /* set, get dynamic field */ + m = rte_pktmbuf_alloc(pktmbuf_pool); + if (m == NULL) + GOTO_FAIL("Cannot allocate mbuf"); + + *RTE_MBUF_DYNFIELD(m, offset, uint8_t *) = 1; + if (*RTE_MBUF_DYNFIELD(m, offset, uint8_t *) != 1) + GOTO_FAIL("failed to read dynamic field"); + *RTE_MBUF_DYNFIELD(m, offset2, uint16_t *) = 1000; + if (*RTE_MBUF_DYNFIELD(m, offset2, uint16_t *) != 1000) + GOTO_FAIL("failed to read dynamic field"); + + /* set a dynamic flag */ + m->ol_flags |= (1ULL << flag); + + rte_mbuf_dyn_dump(stdout); + rte_pktmbuf_free(m); + return 0; +fail: + rte_pktmbuf_free(m); + return -1; +} + static int test_mbuf(void) { @@ -2431,6 +2568,12 @@ test_mbuf(void) goto err; } + /* test registration of dynamic fields and flags */ + if (test_mbuf_dyn(pktmbuf_pool) < 0) { + printf("mbuf dynflag test failed\n"); + goto err; + } + /* create a specific pktmbuf pool with a priv_size != 0 and no data * room size */ pktmbuf_pool2 = rte_pktmbuf_pool_create("test_pktmbuf_pool2", diff --git a/doc/guides/rel_notes/release_19_11.rst b/doc/guides/rel_notes/release_19_11.rst index 2b4cbe6e3b..c14a08af4f 100644 --- a/doc/guides/rel_notes/release_19_11.rst +++ b/doc/guides/rel_notes/release_19_11.rst @@ -65,6 +65,13 @@ New Features The lock-free stack implementation is enabled for aarch64 platforms. +* **Added support of dynamic fields and flags in mbuf.** + + This new feature adds the ability to dynamically register some room + for a field or a flag in the mbuf structure. This is typically used + for specific offload features, where adding a static field or flag + in the mbuf is not justified. + * **Updated the enic driver.** * Added support for Geneve with options offload. diff --git a/lib/librte_eal/common/include/rte_common.h b/lib/librte_eal/common/include/rte_common.h index 7ee94d6988..459d082d14 100644 --- a/lib/librte_eal/common/include/rte_common.h +++ b/lib/librte_eal/common/include/rte_common.h @@ -675,6 +675,18 @@ rte_log2_u64(uint64_t v) }) #endif +/** + * Get the size of a field in a structure. + * + * @param type + * The type of the structure. + * @param field + * The field in the structure. + * @return + * The size of the field in the structure, in bytes. + */ +#define RTE_SIZEOF_FIELD(type, field) (sizeof(((type *)0)->field)) + #define _RTE_STR(x) #x /** Take a macro value and get a string version of it */ #define RTE_STR(x) _RTE_STR(x) diff --git a/lib/librte_mbuf/Makefile b/lib/librte_mbuf/Makefile index f3b76ad23f..019c8dd8ff 100644 --- a/lib/librte_mbuf/Makefile +++ b/lib/librte_mbuf/Makefile @@ -17,11 +17,13 @@ LIBABIVER := 5 # all source are stored in SRCS-y SRCS-$(CONFIG_RTE_LIBRTE_MBUF) := rte_mbuf.c rte_mbuf_ptype.c rte_mbuf_pool_ops.c +SRCS-$(CONFIG_RTE_LIBRTE_MBUF) += rte_mbuf_dyn.c # install includes SYMLINK-$(CONFIG_RTE_LIBRTE_MBUF)-include := rte_mbuf.h SYMLINK-$(CONFIG_RTE_LIBRTE_MBUF)-include += rte_mbuf_core.h SYMLINK-$(CONFIG_RTE_LIBRTE_MBUF)-include += rte_mbuf_ptype.h SYMLINK-$(CONFIG_RTE_LIBRTE_MBUF)-include += rte_mbuf_pool_ops.h +SYMLINK-$(CONFIG_RTE_LIBRTE_MBUF)-include += rte_mbuf_dyn.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/lib/librte_mbuf/meson.build b/lib/librte_mbuf/meson.build index 36bb6eb9d3..59fd07224d 100644 --- a/lib/librte_mbuf/meson.build +++ b/lib/librte_mbuf/meson.build @@ -2,9 +2,11 @@ # Copyright(c) 2017 Intel Corporation version = 5 -sources = files('rte_mbuf.c', 'rte_mbuf_ptype.c', 'rte_mbuf_pool_ops.c') +sources = files('rte_mbuf.c', 'rte_mbuf_ptype.c', 'rte_mbuf_pool_ops.c', + 'rte_mbuf_dyn.c') headers = files('rte_mbuf.h', 'rte_mbuf_core.h', - 'rte_mbuf_ptype.h', 'rte_mbuf_pool_ops.h') + 'rte_mbuf_ptype.h', 'rte_mbuf_pool_ops.h', + 'rte_mbuf_dyn.h') deps += ['mempool'] allow_experimental_apis = true diff --git a/lib/librte_mbuf/rte_mbuf.h b/lib/librte_mbuf/rte_mbuf.h index bd26764a25..92d81972ab 100644 --- a/lib/librte_mbuf/rte_mbuf.h +++ b/lib/librte_mbuf/rte_mbuf.h @@ -1000,6 +1000,20 @@ rte_pktmbuf_attach_extbuf(struct rte_mbuf *m, void *buf_addr, */ #define rte_pktmbuf_detach_extbuf(m) rte_pktmbuf_detach(m) +/** + * Copy dynamic fields from msrc to mdst. + * + * @param mdst + * The destination mbuf. + * @param msrc + * The source mbuf. + */ +static inline void +rte_mbuf_dynfield_copy(struct rte_mbuf *mdst, const struct rte_mbuf *msrc) +{ + memcpy(&mdst->dynfield1, msrc->dynfield1, sizeof(mdst->dynfield1)); +} + /* internal */ static inline void __rte_pktmbuf_copy_hdr(struct rte_mbuf *mdst, const struct rte_mbuf *msrc) @@ -1011,6 +1025,7 @@ __rte_pktmbuf_copy_hdr(struct rte_mbuf *mdst, const struct rte_mbuf *msrc) mdst->hash = msrc->hash; mdst->packet_type = msrc->packet_type; mdst->timestamp = msrc->timestamp; + rte_mbuf_dynfield_copy(mdst, msrc); } /** diff --git a/lib/librte_mbuf/rte_mbuf_core.h b/lib/librte_mbuf/rte_mbuf_core.h index 3398c12c84..3022701461 100644 --- a/lib/librte_mbuf/rte_mbuf_core.h +++ b/lib/librte_mbuf/rte_mbuf_core.h @@ -184,9 +184,12 @@ extern "C" { #define PKT_RX_OUTER_L4_CKSUM_GOOD (1ULL << 22) #define PKT_RX_OUTER_L4_CKSUM_INVALID ((1ULL << 21) | (1ULL << 22)) -/* add new RX flags here */ +/* add new RX flags here, don't forget to update PKT_FIRST_FREE */ -/* add new TX flags here */ +#define PKT_FIRST_FREE (1ULL << 23) +#define PKT_LAST_FREE (1ULL << 39) + +/* add new TX flags here, don't forget to update PKT_LAST_FREE */ /** * Indicate that the metadata field in the mbuf is in use. @@ -689,6 +692,7 @@ struct rte_mbuf { */ struct rte_mbuf_ext_shared_info *shinfo; + uint64_t dynfield1[2]; /**< Reserved for dynamic fields. */ } __rte_cache_aligned; /** diff --git a/lib/librte_mbuf/rte_mbuf_dyn.c b/lib/librte_mbuf/rte_mbuf_dyn.c new file mode 100644 index 0000000000..d6931f8471 --- /dev/null +++ b/lib/librte_mbuf/rte_mbuf_dyn.c @@ -0,0 +1,553 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2019 6WIND S.A. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define RTE_MBUF_DYN_MZNAME "rte_mbuf_dyn" + +struct mbuf_dynfield_elt { + TAILQ_ENTRY(mbuf_dynfield_elt) next; + struct rte_mbuf_dynfield params; + size_t offset; +}; +TAILQ_HEAD(mbuf_dynfield_list, rte_tailq_entry); + +static struct rte_tailq_elem mbuf_dynfield_tailq = { + .name = "RTE_MBUF_DYNFIELD", +}; +EAL_REGISTER_TAILQ(mbuf_dynfield_tailq); + +struct mbuf_dynflag_elt { + TAILQ_ENTRY(mbuf_dynflag_elt) next; + struct rte_mbuf_dynflag params; + unsigned int bitnum; +}; +TAILQ_HEAD(mbuf_dynflag_list, rte_tailq_entry); + +static struct rte_tailq_elem mbuf_dynflag_tailq = { + .name = "RTE_MBUF_DYNFLAG", +}; +EAL_REGISTER_TAILQ(mbuf_dynflag_tailq); + +struct mbuf_dyn_shm { + /** + * For each mbuf byte, free_space[i] != 0 if space is free. + * The value is the size of the biggest aligned element that + * can fit in the zone. + */ + uint8_t free_space[sizeof(struct rte_mbuf)]; + /** Bitfield of available flags. */ + uint64_t free_flags; +}; +static struct mbuf_dyn_shm *shm; + +/* Set the value of free_space[] according to the size and alignment of + * the free areas. This helps to select the best place when reserving a + * dynamic field. Assume tailq is locked. + */ +static void +process_score(void) +{ + size_t off, align, size, i; + + /* first, erase previous info */ + for (i = 0; i < sizeof(struct rte_mbuf); i++) { + if (shm->free_space[i]) + shm->free_space[i] = 1; + } + + for (off = 0; off < sizeof(struct rte_mbuf); off++) { + /* get the size of the free zone */ + for (size = 0; shm->free_space[off + size]; size++) + ; + if (size == 0) + continue; + + /* get the alignment of biggest object that can fit in + * the zone at this offset. + */ + for (align = 1; + (off % (align << 1)) == 0 && (align << 1) <= size; + align <<= 1) + ; + + /* save it in free_space[] */ + for (i = off; i < off + size; i++) + shm->free_space[i] = RTE_MAX(align, shm->free_space[i]); + } +} + +/* Mark the area occupied by a mbuf field as available in the shm. */ +#define mark_free(field) \ + memset(&shm->free_space[offsetof(struct rte_mbuf, field)], \ + 1, sizeof(((struct rte_mbuf *)0)->field)) + +/* Allocate and initialize the shared memory. Assume tailq is locked */ +static int +init_shared_mem(void) +{ + const struct rte_memzone *mz; + uint64_t mask; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + mz = rte_memzone_reserve_aligned(RTE_MBUF_DYN_MZNAME, + sizeof(struct mbuf_dyn_shm), + SOCKET_ID_ANY, 0, + RTE_CACHE_LINE_SIZE); + } else { + mz = rte_memzone_lookup(RTE_MBUF_DYN_MZNAME); + } + if (mz == NULL) + return -1; + + shm = mz->addr; + + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + /* init free_space, keep it sync'd with + * rte_mbuf_dynfield_copy(). + */ + memset(shm, 0, sizeof(*shm)); + mark_free(dynfield1); + + /* init free_flags */ + for (mask = PKT_FIRST_FREE; mask <= PKT_LAST_FREE; mask <<= 1) + shm->free_flags |= mask; + + process_score(); + } + + return 0; +} + +/* check if this offset can be used */ +static int +check_offset(size_t offset, size_t size, size_t align) +{ + size_t i; + + if ((offset & (align - 1)) != 0) + return -1; + if (offset + size > sizeof(struct rte_mbuf)) + return -1; + + for (i = 0; i < size; i++) { + if (!shm->free_space[i + offset]) + return -1; + } + + return 0; +} + +/* assume tailq is locked */ +static struct mbuf_dynfield_elt * +__mbuf_dynfield_lookup(const char *name) +{ + struct mbuf_dynfield_list *mbuf_dynfield_list; + struct mbuf_dynfield_elt *mbuf_dynfield; + struct rte_tailq_entry *te; + + mbuf_dynfield_list = RTE_TAILQ_CAST( + mbuf_dynfield_tailq.head, mbuf_dynfield_list); + + TAILQ_FOREACH(te, mbuf_dynfield_list, next) { + mbuf_dynfield = (struct mbuf_dynfield_elt *)te->data; + if (strcmp(name, mbuf_dynfield->params.name) == 0) + break; + } + + if (te == NULL) { + rte_errno = ENOENT; + return NULL; + } + + return mbuf_dynfield; +} + +int +rte_mbuf_dynfield_lookup(const char *name, struct rte_mbuf_dynfield *params) +{ + struct mbuf_dynfield_elt *mbuf_dynfield; + + if (shm == NULL) { + rte_errno = ENOENT; + return -1; + } + + rte_mcfg_tailq_read_lock(); + mbuf_dynfield = __mbuf_dynfield_lookup(name); + rte_mcfg_tailq_read_unlock(); + + if (mbuf_dynfield == NULL) { + rte_errno = ENOENT; + return -1; + } + + if (params != NULL) + memcpy(params, &mbuf_dynfield->params, sizeof(*params)); + + return mbuf_dynfield->offset; +} + +static int mbuf_dynfield_cmp(const struct rte_mbuf_dynfield *params1, + const struct rte_mbuf_dynfield *params2) +{ + if (strcmp(params1->name, params2->name)) + return -1; + if (params1->size != params2->size) + return -1; + if (params1->align != params2->align) + return -1; + if (params1->flags != params2->flags) + return -1; + return 0; +} + +/* assume tailq is locked */ +static int +__rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params, + size_t req) +{ + struct mbuf_dynfield_list *mbuf_dynfield_list; + struct mbuf_dynfield_elt *mbuf_dynfield = NULL; + struct rte_tailq_entry *te = NULL; + unsigned int best_zone = UINT_MAX; + size_t i, offset; + int ret; + + if (shm == NULL && init_shared_mem() < 0) + return -1; + + mbuf_dynfield = __mbuf_dynfield_lookup(params->name); + if (mbuf_dynfield != NULL) { + if (req != SIZE_MAX && req != mbuf_dynfield->offset) { + rte_errno = EEXIST; + return -1; + } + if (mbuf_dynfield_cmp(params, &mbuf_dynfield->params) < 0) { + rte_errno = EEXIST; + return -1; + } + return mbuf_dynfield->offset; + } + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + rte_errno = EPERM; + return -1; + } + + if (req == SIZE_MAX) { + /* Find the best place to put this field: we search the + * lowest value of shm->free_space[offset]: the zones + * containing room for larger fields are kept for later. + */ + for (offset = 0; + offset < sizeof(struct rte_mbuf); + offset++) { + if (check_offset(offset, params->size, + params->align) == 0 && + shm->free_space[offset] < best_zone) { + best_zone = shm->free_space[offset]; + req = offset; + } + } + if (req == SIZE_MAX) { + rte_errno = ENOENT; + return -1; + } + } else { + if (check_offset(req, params->size, params->align) < 0) { + rte_errno = EBUSY; + return -1; + } + } + + offset = req; + mbuf_dynfield_list = RTE_TAILQ_CAST( + mbuf_dynfield_tailq.head, mbuf_dynfield_list); + + te = rte_zmalloc("MBUF_DYNFIELD_TAILQ_ENTRY", sizeof(*te), 0); + if (te == NULL) + return -1; + + mbuf_dynfield = rte_zmalloc("mbuf_dynfield", sizeof(*mbuf_dynfield), 0); + if (mbuf_dynfield == NULL) { + rte_free(te); + return -1; + } + + ret = strlcpy(mbuf_dynfield->params.name, params->name, + sizeof(mbuf_dynfield->params.name)); + if (ret < 0 || ret >= (int)sizeof(mbuf_dynfield->params.name)) { + rte_errno = ENAMETOOLONG; + rte_free(mbuf_dynfield); + rte_free(te); + return -1; + } + memcpy(&mbuf_dynfield->params, params, sizeof(mbuf_dynfield->params)); + mbuf_dynfield->offset = offset; + te->data = mbuf_dynfield; + + TAILQ_INSERT_TAIL(mbuf_dynfield_list, te, next); + + for (i = offset; i < offset + params->size; i++) + shm->free_space[i] = 0; + process_score(); + + RTE_LOG(DEBUG, MBUF, "Registered dynamic field %s (sz=%zu, al=%zu, fl=0x%x) -> %zd\n", + params->name, params->size, params->align, params->flags, + offset); + + return offset; +} + +int +rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params, + size_t req) +{ + int ret; + + if (params->size >= sizeof(struct rte_mbuf)) { + rte_errno = EINVAL; + return -1; + } + if (!rte_is_power_of_2(params->align)) { + rte_errno = EINVAL; + return -1; + } + if (params->flags != 0) { + rte_errno = EINVAL; + return -1; + } + + rte_mcfg_tailq_write_lock(); + ret = __rte_mbuf_dynfield_register_offset(params, req); + rte_mcfg_tailq_write_unlock(); + + return ret; +} + +int +rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params) +{ + return rte_mbuf_dynfield_register_offset(params, SIZE_MAX); +} + +/* assume tailq is locked */ +static struct mbuf_dynflag_elt * +__mbuf_dynflag_lookup(const char *name) +{ + struct mbuf_dynflag_list *mbuf_dynflag_list; + struct mbuf_dynflag_elt *mbuf_dynflag; + struct rte_tailq_entry *te; + + mbuf_dynflag_list = RTE_TAILQ_CAST( + mbuf_dynflag_tailq.head, mbuf_dynflag_list); + + TAILQ_FOREACH(te, mbuf_dynflag_list, next) { + mbuf_dynflag = (struct mbuf_dynflag_elt *)te->data; + if (strncmp(name, mbuf_dynflag->params.name, + RTE_MBUF_DYN_NAMESIZE) == 0) + break; + } + + if (te == NULL) { + rte_errno = ENOENT; + return NULL; + } + + return mbuf_dynflag; +} + +int +rte_mbuf_dynflag_lookup(const char *name, + struct rte_mbuf_dynflag *params) +{ + struct mbuf_dynflag_elt *mbuf_dynflag; + + if (shm == NULL) { + rte_errno = ENOENT; + return -1; + } + + rte_mcfg_tailq_read_lock(); + mbuf_dynflag = __mbuf_dynflag_lookup(name); + rte_mcfg_tailq_read_unlock(); + + if (mbuf_dynflag == NULL) { + rte_errno = ENOENT; + return -1; + } + + if (params != NULL) + memcpy(params, &mbuf_dynflag->params, sizeof(*params)); + + return mbuf_dynflag->bitnum; +} + +static int mbuf_dynflag_cmp(const struct rte_mbuf_dynflag *params1, + const struct rte_mbuf_dynflag *params2) +{ + if (strcmp(params1->name, params2->name)) + return -1; + if (params1->flags != params2->flags) + return -1; + return 0; +} + +/* assume tailq is locked */ +static int +__rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params, + unsigned int req) +{ + struct mbuf_dynflag_list *mbuf_dynflag_list; + struct mbuf_dynflag_elt *mbuf_dynflag = NULL; + struct rte_tailq_entry *te = NULL; + unsigned int bitnum; + int ret; + + if (shm == NULL && init_shared_mem() < 0) + return -1; + + mbuf_dynflag = __mbuf_dynflag_lookup(params->name); + if (mbuf_dynflag != NULL) { + if (req != UINT_MAX && req != mbuf_dynflag->bitnum) { + rte_errno = EEXIST; + return -1; + } + if (mbuf_dynflag_cmp(params, &mbuf_dynflag->params) < 0) { + rte_errno = EEXIST; + return -1; + } + return mbuf_dynflag->bitnum; + } + + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + rte_errno = EPERM; + return -1; + } + + if (req == UINT_MAX) { + if (shm->free_flags == 0) { + rte_errno = ENOENT; + return -1; + } + bitnum = rte_bsf64(shm->free_flags); + } else { + if ((shm->free_flags & (1ULL << req)) == 0) { + rte_errno = EBUSY; + return -1; + } + bitnum = req; + } + + mbuf_dynflag_list = RTE_TAILQ_CAST( + mbuf_dynflag_tailq.head, mbuf_dynflag_list); + + te = rte_zmalloc("MBUF_DYNFLAG_TAILQ_ENTRY", sizeof(*te), 0); + if (te == NULL) + return -1; + + mbuf_dynflag = rte_zmalloc("mbuf_dynflag", sizeof(*mbuf_dynflag), 0); + if (mbuf_dynflag == NULL) { + rte_free(te); + return -1; + } + + ret = strlcpy(mbuf_dynflag->params.name, params->name, + sizeof(mbuf_dynflag->params.name)); + if (ret < 0 || ret >= (int)sizeof(mbuf_dynflag->params.name)) { + rte_free(mbuf_dynflag); + rte_free(te); + rte_errno = ENAMETOOLONG; + return -1; + } + mbuf_dynflag->bitnum = bitnum; + te->data = mbuf_dynflag; + + TAILQ_INSERT_TAIL(mbuf_dynflag_list, te, next); + + shm->free_flags &= ~(1ULL << bitnum); + + RTE_LOG(DEBUG, MBUF, "Registered dynamic flag %s (fl=0x%x) -> %u\n", + params->name, params->flags, bitnum); + + return bitnum; +} + +int +rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params, + unsigned int req) +{ + int ret; + + if (req >= RTE_SIZEOF_FIELD(struct rte_mbuf, ol_flags) * CHAR_BIT && + req != UINT_MAX) { + rte_errno = EINVAL; + return -1; + } + + rte_mcfg_tailq_write_lock(); + ret = __rte_mbuf_dynflag_register_bitnum(params, req); + rte_mcfg_tailq_write_unlock(); + + return ret; +} + +int +rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params) +{ + return rte_mbuf_dynflag_register_bitnum(params, UINT_MAX); +} + +void rte_mbuf_dyn_dump(FILE *out) +{ + struct mbuf_dynfield_list *mbuf_dynfield_list; + struct mbuf_dynfield_elt *dynfield; + struct mbuf_dynflag_list *mbuf_dynflag_list; + struct mbuf_dynflag_elt *dynflag; + struct rte_tailq_entry *te; + size_t i; + + rte_mcfg_tailq_write_lock(); + init_shared_mem(); + fprintf(out, "Reserved fields:\n"); + mbuf_dynfield_list = RTE_TAILQ_CAST( + mbuf_dynfield_tailq.head, mbuf_dynfield_list); + TAILQ_FOREACH(te, mbuf_dynfield_list, next) { + dynfield = (struct mbuf_dynfield_elt *)te->data; + fprintf(out, " name=%s offset=%zd size=%zd align=%zd flags=%x\n", + dynfield->params.name, dynfield->offset, + dynfield->params.size, dynfield->params.align, + dynfield->params.flags); + } + fprintf(out, "Reserved flags:\n"); + mbuf_dynflag_list = RTE_TAILQ_CAST( + mbuf_dynflag_tailq.head, mbuf_dynflag_list); + TAILQ_FOREACH(te, mbuf_dynflag_list, next) { + dynflag = (struct mbuf_dynflag_elt *)te->data; + fprintf(out, " name=%s bitnum=%u flags=%x\n", + dynflag->params.name, dynflag->bitnum, + dynflag->params.flags); + } + fprintf(out, "Free space in mbuf (0 = free, value = zone alignment):\n"); + for (i = 0; i < sizeof(struct rte_mbuf); i++) { + if ((i % 8) == 0) + fprintf(out, " %4.4zx: ", i); + fprintf(out, "%2.2x%s", shm->free_space[i], + (i % 8 != 7) ? " " : "\n"); + } + rte_mcfg_tailq_write_unlock(); +} diff --git a/lib/librte_mbuf/rte_mbuf_dyn.h b/lib/librte_mbuf/rte_mbuf_dyn.h new file mode 100644 index 0000000000..2e9d418cfe --- /dev/null +++ b/lib/librte_mbuf/rte_mbuf_dyn.h @@ -0,0 +1,239 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2019 6WIND S.A. + */ + +#ifndef _RTE_MBUF_DYN_H_ +#define _RTE_MBUF_DYN_H_ + +/** + * @file + * RTE Mbuf dynamic fields and flags + * + * Many DPDK features require to store data inside the mbuf. As the room + * in mbuf structure is limited, it is not possible to have a field for + * each feature. Also, changing fields in the mbuf structure can break + * the API or ABI. + * + * This module addresses this issue, by enabling the dynamic + * registration of fields or flags: + * + * - a dynamic field is a named area in the rte_mbuf structure, with a + * given size (>= 1 byte) and alignment constraint. + * - a dynamic flag is a named bit in the rte_mbuf structure, stored + * in mbuf->ol_flags. + * + * The placement of the field or flag can be automatic, in this case the + * zones that have the smallest size and alignment constraint are + * selected in priority. Else, a specific field offset or flag bit + * number can be requested through the API. + * + * The typical use case is when a specific offload feature requires to + * register a dedicated offload field in the mbuf structure, and adding + * a static field or flag is not justified. + * + * Example of use: + * + * - A rte_mbuf_dynfield structure is defined, containing the parameters + * of the dynamic field to be registered: + * const struct rte_mbuf_dynfield rte_dynfield_my_feature = { ... }; + * - The application initializes the PMD, and asks for this feature + * at port initialization by passing DEV_RX_OFFLOAD_MY_FEATURE in + * rxconf. This will make the PMD to register the field by calling + * rte_mbuf_dynfield_register(&rte_dynfield_my_feature). The PMD + * stores the returned offset. + * - The application that uses the offload feature also registers + * the field to retrieve the same offset. + * - When the PMD receives a packet, it can set the field: + * *RTE_MBUF_DYNFIELD(m, offset, ) = value; + * - In the main loop, the application can retrieve the value with + * the same macro. + * + * To avoid wasting space, the dynamic fields or flags must only be + * reserved on demand, when an application asks for the related feature. + * + * The registration can be done at any moment, but it is not possible + * to unregister fields or flags for now. + * + * A dynamic field can be reserved and used by an application only. + * It can for instance be a packet mark. + * + * To avoid namespace collisions, the dynamic mbuf field or flag names + * have to be chosen with care. It is advised to use the same + * conventions than function names in dpdk: + * - "rte_mbuf_dynfield_" if defined in mbuf library + * - "rte__dynfield_" if defined in another library + * - "rte_net__dynfield_" if defined in a in PMD + * - any name that does not start with "rte_" in an application + */ + +#include +/** + * Maximum length of the dynamic field or flag string. + */ +#define RTE_MBUF_DYN_NAMESIZE 64 + +/** + * Structure describing the parameters of a mbuf dynamic field. + */ +struct rte_mbuf_dynfield { + char name[RTE_MBUF_DYN_NAMESIZE]; /**< Name of the field. */ + size_t size; /**< The number of bytes to reserve. */ + size_t align; /**< The alignment constraint (power of 2). */ + unsigned int flags; /**< Reserved for future use, must be 0. */ +}; + +/** + * Structure describing the parameters of a mbuf dynamic flag. + */ +struct rte_mbuf_dynflag { + char name[RTE_MBUF_DYN_NAMESIZE]; /**< Name of the dynamic flag. */ + unsigned int flags; /**< Reserved for future use, must be 0. */ +}; + +/** + * Register space for a dynamic field in the mbuf structure. + * + * If the field is already registered (same name and parameters), its + * offset is returned. + * + * @param params + * A structure containing the requested parameters (name, size, + * alignment constraint and flags). + * @return + * The offset in the mbuf structure, or -1 on error. + * Possible values for rte_errno: + * - EINVAL: invalid parameters (size, align, or flags). + * - EEXIST: this name is already register with different parameters. + * - EPERM: called from a secondary process. + * - ENOENT: not enough room in mbuf. + * - ENOMEM: allocation failure. + * - ENAMETOOLONG: name does not ends with \0. + */ +__rte_experimental +int rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params); + +/** + * Register space for a dynamic field in the mbuf structure at offset. + * + * If the field is already registered (same name, parameters and offset), + * the offset is returned. + * + * @param params + * A structure containing the requested parameters (name, size, + * alignment constraint and flags). + * @param offset + * The requested offset. Ignored if SIZE_MAX is passed. + * @return + * The offset in the mbuf structure, or -1 on error. + * Possible values for rte_errno: + * - EINVAL: invalid parameters (size, align, flags, or offset). + * - EEXIST: this name is already register with different parameters. + * - EBUSY: the requested offset cannot be used. + * - EPERM: called from a secondary process. + * - ENOENT: not enough room in mbuf. + * - ENOMEM: allocation failure. + * - ENAMETOOLONG: name does not ends with \0. + */ +__rte_experimental +int rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params, + size_t offset); + +/** + * Lookup for a registered dynamic mbuf field. + * + * @param name + * A string identifying the dynamic field. + * @param params + * If not NULL, and if the lookup is successful, the structure is + * filled with the parameters of the dynamic field. + * @return + * The offset of this field in the mbuf structure, or -1 on error. + * Possible values for rte_errno: + * - ENOENT: no dynamic field matches this name. + */ +__rte_experimental +int rte_mbuf_dynfield_lookup(const char *name, + struct rte_mbuf_dynfield *params); + +/** + * Register a dynamic flag in the mbuf structure. + * + * If the flag is already registered (same name and parameters), its + * bitnum is returned. + * + * @param params + * A structure containing the requested parameters of the dynamic + * flag (name and options). + * @return + * The number of the reserved bit, or -1 on error. + * Possible values for rte_errno: + * - EINVAL: invalid parameters (size, align, or flags). + * - EEXIST: this name is already register with different parameters. + * - EPERM: called from a secondary process. + * - ENOENT: no more flag available. + * - ENOMEM: allocation failure. + * - ENAMETOOLONG: name is longer than RTE_MBUF_DYN_NAMESIZE - 1. + */ +__rte_experimental +int rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params); + +/** + * Register a dynamic flag in the mbuf structure specifying bitnum. + * + * If the flag is already registered (same name, parameters and bitnum), + * the bitnum is returned. + * + * @param params + * A structure containing the requested parameters of the dynamic + * flag (name and options). + * @param bitnum + * The requested bitnum. Ignored if UINT_MAX is passed. + * @return + * The number of the reserved bit, or -1 on error. + * Possible values for rte_errno: + * - EINVAL: invalid parameters (size, align, or flags). + * - EEXIST: this name is already register with different parameters. + * - EBUSY: the requested bitnum cannot be used. + * - EPERM: called from a secondary process. + * - ENOENT: no more flag available. + * - ENOMEM: allocation failure. + * - ENAMETOOLONG: name is longer than RTE_MBUF_DYN_NAMESIZE - 1. + */ +__rte_experimental +int rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params, + unsigned int bitnum); + +/** + * Lookup for a registered dynamic mbuf flag. + * + * @param name + * A string identifying the dynamic flag. + * @param params + * If not NULL, and if the lookup is successful, the structure is + * filled with the parameters of the dynamic flag. + * @return + * The offset of this flag in the mbuf structure, or -1 on error. + * Possible values for rte_errno: + * - ENOENT: no dynamic flag matches this name. + */ +__rte_experimental +int rte_mbuf_dynflag_lookup(const char *name, + struct rte_mbuf_dynflag *params); + +/** + * Helper macro to access to a dynamic field. + */ +#define RTE_MBUF_DYNFIELD(m, offset, type) ((type)((uintptr_t)(m) + (offset))) + +/** + * Dump the status of dynamic fields and flags. + * + * @param out + * The stream where the status is displayed. + */ +__rte_experimental +void rte_mbuf_dyn_dump(FILE *out); + +/* Placeholder for dynamic fields and flags declarations. */ + +#endif diff --git a/lib/librte_mbuf/rte_mbuf_version.map b/lib/librte_mbuf/rte_mbuf_version.map index a4f41d7fd3..263dc0a21e 100644 --- a/lib/librte_mbuf/rte_mbuf_version.map +++ b/lib/librte_mbuf/rte_mbuf_version.map @@ -58,6 +58,13 @@ EXPERIMENTAL { global: rte_mbuf_check; + rte_mbuf_dynfield_lookup; + rte_mbuf_dynfield_register; + rte_mbuf_dynfield_register_offset; + rte_mbuf_dynflag_lookup; + rte_mbuf_dynflag_register; + rte_mbuf_dynflag_register_bitnum; + rte_mbuf_dyn_dump; rte_pktmbuf_copy; rte_pktmbuf_free_bulk;