d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

mempool: add namespace to internal helpers

Browse Source 
Add RTE_ prefix to internal API defined in public header.
Use the prefix instead of double underscore.
Use uppercase for macros in the case of name conflict.

Signed-off-by: Andrew Rybchenko <andrew.rybchenko@oktetlabs.ru>
Acked-by: Olivier Matz <olivier.matz@6wind.com>
This commit is contained in:
Andrew Rybchenko 2021-10-19 20:40:19 +03:00 committed by David Marchand
parent c47d7b90a1
commit ad276d5c7e
13 changed files with 103 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/event/octeontx/ssovf_worker.h
View File

@ -83,7 +83,7 @@ ssovf_octeontx_wqe_xtract_mseg(octtx_wqe_t *wqe,
mbuf->data_off = sizeof(octtx_pki_buflink_t);
__mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1);
if (nb_segs == 1)
mbuf->data_len = bytes_left;
else

12
drivers/net/cnxk/cn10k_rx.h
View File

@ -276,7 +276,7 @@ nix_cqe_xtract_mseg(const union nix_rx_parse_u *rx, struct rte_mbuf *mbuf,
mbuf->next = ((struct rte_mbuf *)*iova_list) - 1;
mbuf = mbuf->next;
__mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1);
mbuf->data_len = sg & 0xFFFF;
sg = sg >> 16;
@ -306,7 +306,7 @@ cn10k_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag,
uint64_t ol_flags = 0;
/* Mark mempool obj as "get" as it is alloc'ed by NIX */
__mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1);
if (flag & NIX_RX_OFFLOAD_PTYPE_F)
mbuf->packet_type = nix_ptype_get(lookup_mem, w1);
@ -905,10 +905,10 @@ cn10k_nix_recv_pkts_vector(void *args, struct rte_mbuf **mbufs, uint16_t pkts,
roc_prefetch_store_keep(mbuf3);
/* Mark mempool obj as "get" as it is alloc'ed by NIX */
__mempool_check_cookies(mbuf0->pool, (void **)&mbuf0, 1, 1);
__mempool_check_cookies(mbuf1->pool, (void **)&mbuf1, 1, 1);
__mempool_check_cookies(mbuf2->pool, (void **)&mbuf2, 1, 1);
__mempool_check_cookies(mbuf3->pool, (void **)&mbuf3, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf0->pool, (void **)&mbuf0, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf1->pool, (void **)&mbuf1, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf2->pool, (void **)&mbuf2, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf3->pool, (void **)&mbuf3, 1, 1);
packets += NIX_DESCS_PER_LOOP;

30
drivers/net/cnxk/cn10k_tx.h
View File

@ -677,7 +677,7 @@ cn10k_nix_xmit_prepare(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags,
}
/* Mark mempool object as "put" since it is freed by NIX */
if (!send_hdr->w0.df)
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
} else {
sg->seg1_size = m->data_len;
*(rte_iova_t *)(sg + 1) = rte_mbuf_data_iova(m);
@ -789,7 +789,7 @@ cn10k_nix_prepare_mseg(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags)
/* Mark mempool object as "put" since it is freed by NIX */
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
if (!(sg_u & (1ULL << 55)))
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
rte_io_wmb();
#endif
m = m_next;
@ -808,7 +808,7 @@ cn10k_nix_prepare_mseg(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags)
*/
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
if (!(sg_u & (1ULL << (i + 55))))
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
#endif
slist++;
i++;
@ -1177,7 +1177,7 @@ cn10k_nix_prepare_mseg_vec_list(struct rte_mbuf *m, uint64_t *cmd,
/* Mark mempool object as "put" since it is freed by NIX */
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
if (!(sg_u & (1ULL << 55)))
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
rte_io_wmb();
#endif
@ -1194,7 +1194,7 @@ cn10k_nix_prepare_mseg_vec_list(struct rte_mbuf *m, uint64_t *cmd,
*/
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
if (!(sg_u & (1ULL << (i + 55))))
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
rte_io_wmb();
#endif
slist++;
@ -1235,7 +1235,7 @@ cn10k_nix_prepare_mseg_vec(struct rte_mbuf *m, uint64_t *cmd, uint64x2_t *cmd0,
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
sg.u = vgetq_lane_u64(cmd1[0], 0);
if (!(sg.u & (1ULL << 55)))
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
rte_io_wmb();
#endif
return;
@ -1425,7 +1425,7 @@ cn10k_nix_xmit_store(struct rte_mbuf *mbuf, uint8_t segdw, uintptr_t laddr,
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
sg.u = vgetq_lane_u64(cmd1, 0);
if (!(sg.u & (1ULL << 55)))
__mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1,
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1,
0);
rte_io_wmb();
#endif
@ -2352,28 +2352,28 @@ again:
if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf0))
vsetq_lane_u64(0x80000, xmask01, 0);
else
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf0)->pool,
(void **)&mbuf0, 1, 0);
if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf1))
vsetq_lane_u64(0x80000, xmask01, 1);
else
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf1)->pool,
(void **)&mbuf1, 1, 0);
if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf2))
vsetq_lane_u64(0x80000, xmask23, 0);
else
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf2)->pool,
(void **)&mbuf2, 1, 0);
if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf3))
vsetq_lane_u64(0x80000, xmask23, 1);
else
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf3)->pool,
(void **)&mbuf3, 1, 0);
senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01);
@ -2389,19 +2389,19 @@ again:
/* Mark mempool object as "put" since
* it is freed by NIX
*/
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf0)->pool,
(void **)&mbuf0, 1, 0);
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf1)->pool,
(void **)&mbuf1, 1, 0);
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf2)->pool,
(void **)&mbuf2, 1, 0);
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf3)->pool,
(void **)&mbuf3, 1, 0);
}

12
drivers/net/cnxk/cn9k_rx.h
View File

@ -151,7 +151,7 @@ nix_cqe_xtract_mseg(const union nix_rx_parse_u *rx, struct rte_mbuf *mbuf,
mbuf->next = ((struct rte_mbuf *)*iova_list) - 1;
mbuf = mbuf->next;
__mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1);
mbuf->data_len = sg & 0xFFFF;
sg = sg >> 16;
@ -288,7 +288,7 @@ cn9k_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag,
uint64_t ol_flags = 0;
/* Mark mempool obj as "get" as it is alloc'ed by NIX */
__mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1);
if (flag & NIX_RX_OFFLOAD_PTYPE_F)
packet_type = nix_ptype_get(lookup_mem, w1);
@ -757,10 +757,10 @@ cn9k_nix_recv_pkts_vector(void *rx_queue, struct rte_mbuf **rx_pkts,
roc_prefetch_store_keep(mbuf3);
/* Mark mempool obj as "get" as it is alloc'ed by NIX */
__mempool_check_cookies(mbuf0->pool, (void **)&mbuf0, 1, 1);
__mempool_check_cookies(mbuf1->pool, (void **)&mbuf1, 1, 1);
__mempool_check_cookies(mbuf2->pool, (void **)&mbuf2, 1, 1);
__mempool_check_cookies(mbuf3->pool, (void **)&mbuf3, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf0->pool, (void **)&mbuf0, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf1->pool, (void **)&mbuf1, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf2->pool, (void **)&mbuf2, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf3->pool, (void **)&mbuf3, 1, 1);
/* Advance head pointer and packets */
head += NIX_DESCS_PER_LOOP;

26
drivers/net/cnxk/cn9k_tx.h
View File

@ -285,7 +285,7 @@ cn9k_nix_xmit_prepare(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags,
}
/* Mark mempool object as "put" since it is freed by NIX */
if (!send_hdr->w0.df)
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
}
}
@ -397,7 +397,7 @@ cn9k_nix_prepare_mseg(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags)
/* Mark mempool object as "put" since it is freed by NIX */
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
if (!(sg_u & (1ULL << (i + 55))))
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
rte_io_wmb();
#endif
slist++;
@ -611,7 +611,7 @@ cn9k_nix_prepare_mseg_vec_list(struct rte_mbuf *m, uint64_t *cmd,
/* Mark mempool object as "put" since it is freed by NIX */
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
if (!(sg_u & (1ULL << 55)))
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
rte_io_wmb();
#endif
@ -628,7 +628,7 @@ cn9k_nix_prepare_mseg_vec_list(struct rte_mbuf *m, uint64_t *cmd,
*/
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
if (!(sg_u & (1ULL << (i + 55))))
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
rte_io_wmb();
#endif
slist++;
@ -680,7 +680,7 @@ cn9k_nix_prepare_mseg_vec(struct rte_mbuf *m, uint64_t *cmd, uint64x2_t *cmd0,
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
sg.u = vgetq_lane_u64(cmd1[0], 0);
if (!(sg.u & (1ULL << 55)))
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
rte_io_wmb();
#endif
return 2 + !!(flags & NIX_TX_NEED_EXT_HDR) +
@ -1627,28 +1627,28 @@ cn9k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf0))
vsetq_lane_u64(0x80000, xmask01, 0);
else
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf0)->pool,
(void **)&mbuf0, 1, 0);
if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf1))
vsetq_lane_u64(0x80000, xmask01, 1);
else
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf1)->pool,
(void **)&mbuf1, 1, 0);
if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf2))
vsetq_lane_u64(0x80000, xmask23, 0);
else
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf2)->pool,
(void **)&mbuf2, 1, 0);
if (cnxk_nix_prefree_seg((struct rte_mbuf *)mbuf3))
vsetq_lane_u64(0x80000, xmask23, 1);
else
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf3)->pool,
(void **)&mbuf3, 1, 0);
senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01);
@ -1667,19 +1667,19 @@ cn9k_nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
/* Mark mempool object as "put" since
* it is freed by NIX
*/
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf0)->pool,
(void **)&mbuf0, 1, 0);
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf1)->pool,
(void **)&mbuf1, 1, 0);
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf2)->pool,
(void **)&mbuf2, 1, 0);
__mempool_check_cookies(
RTE_MEMPOOL_CHECK_COOKIES(
((struct rte_mbuf *)mbuf3)->pool,
(void **)&mbuf3, 1, 0);
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG

4
drivers/net/octeontx/octeontx_rxtx.h
View File

@ -344,7 +344,7 @@ __octeontx_xmit_prepare(struct rte_mbuf *tx_pkt, uint64_t *cmd_buf,
/* Mark mempool object as "put" since it is freed by PKO */
if (!(cmd_buf[0] & (1ULL << 58)))
__mempool_check_cookies(m_tofree->pool, (void **)&m_tofree,
RTE_MEMPOOL_CHECK_COOKIES(m_tofree->pool, (void **)&m_tofree,
1, 0);
/* Get the gaura Id */
gaura_id =
@ -417,7 +417,7 @@ __octeontx_xmit_mseg_prepare(struct rte_mbuf *tx_pkt, uint64_t *cmd_buf,
*/
if (!(cmd_buf[nb_desc] & (1ULL << 57))) {
tx_pkt->next = NULL;
__mempool_check_cookies(m_tofree->pool,
RTE_MEMPOOL_CHECK_COOKIES(m_tofree->pool,
(void **)&m_tofree, 1, 0);
}
nb_desc++;

2
drivers/net/octeontx2/otx2_ethdev_sec_tx.h
View File

@ -146,7 +146,7 @@ otx2_sec_event_tx(uint64_t base, struct rte_event *ev, struct rte_mbuf *m,
sd->nix_iova.addr = rte_mbuf_data_iova(m);
/* Mark mempool object as "put" since it is freed by NIX */
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
if (!ev->sched_type)
otx2_ssogws_head_wait(base + SSOW_LF_GWS_TAG);

8
drivers/net/octeontx2/otx2_rx.c
View File

@ -296,10 +296,10 @@ nix_recv_pkts_vector(void *rx_queue, struct rte_mbuf **rx_pkts,
otx2_prefetch_store_keep(mbuf3);
/* Mark mempool obj as "get" as it is alloc'ed by NIX */
__mempool_check_cookies(mbuf0->pool, (void **)&mbuf0, 1, 1);
__mempool_check_cookies(mbuf1->pool, (void **)&mbuf1, 1, 1);
__mempool_check_cookies(mbuf2->pool, (void **)&mbuf2, 1, 1);
__mempool_check_cookies(mbuf3->pool, (void **)&mbuf3, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf0->pool, (void **)&mbuf0, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf1->pool, (void **)&mbuf1, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf2->pool, (void **)&mbuf2, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf3->pool, (void **)&mbuf3, 1, 1);
/* Advance head pointer and packets */
head += NIX_DESCS_PER_LOOP; head &= qmask;

4
drivers/net/octeontx2/otx2_rx.h
View File

@ -199,7 +199,7 @@ nix_cqe_xtract_mseg(const struct nix_rx_parse_s *rx,
mbuf->next = ((struct rte_mbuf *)*iova_list) - 1;
mbuf = mbuf->next;
__mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1);
mbuf->data_len = sg & 0xFFFF;
sg = sg >> 16;
@ -309,7 +309,7 @@ otx2_nix_cqe_to_mbuf(const struct nix_cqe_hdr_s *cq, const uint32_t tag,
uint64_t ol_flags = 0;
/* Mark mempool obj as "get" as it is alloc'ed by NIX */
__mempool_check_cookies(mbuf->pool, (void **)&mbuf, 1, 1);
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf, 1, 1);
if (flag & NIX_RX_OFFLOAD_PTYPE_F)
mbuf->packet_type = nix_ptype_get(lookup_mem, w1);

16
drivers/net/octeontx2/otx2_tx.c
View File

@ -202,7 +202,7 @@ nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
if (otx2_nix_prefree_seg(mbuf))
vsetq_lane_u64(0x80000, xmask01, 0);
else
__mempool_check_cookies(mbuf->pool,
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool,
(void **)&mbuf,
1, 0);
@ -211,7 +211,7 @@ nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
if (otx2_nix_prefree_seg(mbuf))
vsetq_lane_u64(0x80000, xmask01, 1);
else
__mempool_check_cookies(mbuf->pool,
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool,
(void **)&mbuf,
1, 0);
@ -220,7 +220,7 @@ nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
if (otx2_nix_prefree_seg(mbuf))
vsetq_lane_u64(0x80000, xmask23, 0);
else
__mempool_check_cookies(mbuf->pool,
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool,
(void **)&mbuf,
1, 0);
@ -229,7 +229,7 @@ nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
if (otx2_nix_prefree_seg(mbuf))
vsetq_lane_u64(0x80000, xmask23, 1);
else
__mempool_check_cookies(mbuf->pool,
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool,
(void **)&mbuf,
1, 0);
senddesc01_w0 = vorrq_u64(senddesc01_w0, xmask01);
@ -245,22 +245,22 @@ nix_xmit_pkts_vector(void *tx_queue, struct rte_mbuf **tx_pkts,
*/
mbuf = (struct rte_mbuf *)((uintptr_t)mbuf0 -
offsetof(struct rte_mbuf, buf_iova));
__mempool_check_cookies(mbuf->pool, (void **)&mbuf,
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf,
1, 0);
mbuf = (struct rte_mbuf *)((uintptr_t)mbuf1 -
offsetof(struct rte_mbuf, buf_iova));
__mempool_check_cookies(mbuf->pool, (void **)&mbuf,
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf,
1, 0);
mbuf = (struct rte_mbuf *)((uintptr_t)mbuf2 -
offsetof(struct rte_mbuf, buf_iova));
__mempool_check_cookies(mbuf->pool, (void **)&mbuf,
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf,
1, 0);
mbuf = (struct rte_mbuf *)((uintptr_t)mbuf3 -
offsetof(struct rte_mbuf, buf_iova));
__mempool_check_cookies(mbuf->pool, (void **)&mbuf,
RTE_MEMPOOL_CHECK_COOKIES(mbuf->pool, (void **)&mbuf,
1, 0);
RTE_SET_USED(mbuf);
}

4
drivers/net/octeontx2/otx2_tx.h
View File

@ -372,7 +372,7 @@ otx2_nix_xmit_prepare(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags,
}
/* Mark mempool object as "put" since it is freed by NIX */
if (!send_hdr->w0.df)
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
}
}
@ -450,7 +450,7 @@ otx2_nix_prepare_mseg(struct rte_mbuf *m, uint64_t *cmd, const uint16_t flags)
/* Mark mempool object as "put" since it is freed by NIX */
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
if (!(sg_u & (1ULL << (i + 55))))
__mempool_check_cookies(m->pool, (void **)&m, 1, 0);
RTE_MEMPOOL_CHECK_COOKIES(m->pool, (void **)&m, 1, 0);
rte_io_wmb();
#endif
slist++;

8
lib/mempool/rte_mempool.c
View File

@ -179,7 +179,7 @@ mempool_add_elem(struct rte_mempool *mp, __rte_unused void *opaque,
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
hdr->cookie = RTE_MEMPOOL_HEADER_COOKIE2;
tlr = __mempool_get_trailer(obj);
tlr = rte_mempool_get_trailer(obj);
tlr->cookie = RTE_MEMPOOL_TRAILER_COOKIE;
#endif
}
@ -1091,7 +1091,7 @@ void rte_mempool_check_cookies(const struct rte_mempool *mp,
rte_panic("MEMPOOL: object is owned by another "
"mempool\n");
hdr = __mempool_get_header(obj);
hdr = rte_mempool_get_header(obj);
cookie = hdr->cookie;
if (free == 0) {
@ -1119,7 +1119,7 @@ void rte_mempool_check_cookies(const struct rte_mempool *mp,
rte_panic("MEMPOOL: bad header cookie (audit)\n");
}
}
tlr = __mempool_get_trailer(obj);
tlr = rte_mempool_get_trailer(obj);
cookie = tlr->cookie;
if (cookie != RTE_MEMPOOL_TRAILER_COOKIE) {
RTE_LOG(CRIT, MEMPOOL,
@ -1171,7 +1171,7 @@ static void
mempool_obj_audit(struct rte_mempool *mp, __rte_unused void *opaque,
void *obj, __rte_unused unsigned idx)
{
__mempool_check_cookies(mp, &obj, 1, 2);
RTE_MEMPOOL_CHECK_COOKIES(mp, &obj, 1, 2);
}
static void

77
lib/mempool/rte_mempool.h
View File

@ -301,14 +301,14 @@ struct rte_mempool {
* Number to add to the object-oriented statistics.
*/
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
#define __MEMPOOL_STAT_ADD(mp, name, n) do { \
#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do { \
unsigned __lcore_id = rte_lcore_id(); \
if (__lcore_id < RTE_MAX_LCORE) { \
mp->stats[__lcore_id].name += n; \
} \
} while(0)
} while (0)
#else
#define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)
#define RTE_MEMPOOL_STAT_ADD(mp, name, n) do {} while (0)
#endif
/**
@ -324,7 +324,8 @@ struct rte_mempool {
(sizeof(struct rte_mempool_cache) * RTE_MAX_LCORE)))
/* return the header of a mempool object (internal) */
static inline struct rte_mempool_objhdr *__mempool_get_header(void *obj)
static inline struct rte_mempool_objhdr *
rte_mempool_get_header(void *obj)
{
return (struct rte_mempool_objhdr *)RTE_PTR_SUB(obj,
sizeof(struct rte_mempool_objhdr));
@ -341,12 +342,12 @@ static inline struct rte_mempool_objhdr *__mempool_get_header(void *obj)
*/
static inline struct rte_mempool *rte_mempool_from_obj(void *obj)
{
struct rte_mempool_objhdr *hdr = __mempool_get_header(obj);
struct rte_mempool_objhdr *hdr = rte_mempool_get_header(obj);
return hdr->mp;
}
/* return the trailer of a mempool object (internal) */
static inline struct rte_mempool_objtlr *__mempool_get_trailer(void *obj)
static inline struct rte_mempool_objtlr *rte_mempool_get_trailer(void *obj)
{
struct rte_mempool *mp = rte_mempool_from_obj(obj);
return (struct rte_mempool_objtlr *)RTE_PTR_ADD(obj, mp->elt_size);
@ -370,10 +371,10 @@ void rte_mempool_check_cookies(const struct rte_mempool *mp,
void * const *obj_table_const, unsigned n, int free);
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
#define __mempool_check_cookies(mp, obj_table_const, n, free) \
#define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) \
rte_mempool_check_cookies(mp, obj_table_const, n, free)
#else
#define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)
#define RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table_const, n, free) do {} while (0)
#endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
/**
@ -395,13 +396,13 @@ void rte_mempool_contig_blocks_check_cookies(const struct rte_mempool *mp,
void * const *first_obj_table_const, unsigned int n, int free);
#ifdef RTE_LIBRTE_MEMPOOL_DEBUG
#define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
free) \
#define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
free) \
rte_mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
free)
#else
#define __mempool_contig_blocks_check_cookies(mp, first_obj_table_const, n, \
free) \
#define RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table_const, n, \
free) \
do {} while (0)
#endif /* RTE_LIBRTE_MEMPOOL_DEBUG */
@ -736,8 +737,8 @@ rte_mempool_ops_dequeue_bulk(struct rte_mempool *mp,
ops = rte_mempool_get_ops(mp->ops_index);
ret = ops->dequeue(mp, obj_table, n);
if (ret == 0) {
__MEMPOOL_STAT_ADD(mp, get_common_pool_bulk, 1);
__MEMPOOL_STAT_ADD(mp, get_common_pool_objs, n);
RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_bulk, 1);
RTE_MEMPOOL_STAT_ADD(mp, get_common_pool_objs, n);
}
return ret;
}
@ -786,8 +787,8 @@ rte_mempool_ops_enqueue_bulk(struct rte_mempool *mp, void * const *obj_table,
{
struct rte_mempool_ops *ops;
__MEMPOOL_STAT_ADD(mp, put_common_pool_bulk, 1);
__MEMPOOL_STAT_ADD(mp, put_common_pool_objs, n);
RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_bulk, 1);
RTE_MEMPOOL_STAT_ADD(mp, put_common_pool_objs, n);
rte_mempool_trace_ops_enqueue_bulk(mp, obj_table, n);
ops = rte_mempool_get_ops(mp->ops_index);
return ops->enqueue(mp, obj_table, n);
@ -1312,14 +1313,14 @@ rte_mempool_cache_flush(struct rte_mempool_cache *cache,
* A pointer to a mempool cache structure. May be NULL if not needed.
*/
static __rte_always_inline void
__mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
unsigned int n, struct rte_mempool_cache *cache)
rte_mempool_do_generic_put(struct rte_mempool *mp, void * const *obj_table,
unsigned int n, struct rte_mempool_cache *cache)
{
void **cache_objs;
/* increment stat now, adding in mempool always success */
__MEMPOOL_STAT_ADD(mp, put_bulk, 1);
__MEMPOOL_STAT_ADD(mp, put_objs, n);
RTE_MEMPOOL_STAT_ADD(mp, put_bulk, 1);
RTE_MEMPOOL_STAT_ADD(mp, put_objs, n);
/* No cache provided or if put would overflow mem allocated for cache */
if (unlikely(cache == NULL || n > RTE_MEMPOOL_CACHE_MAX_SIZE))
@ -1376,8 +1377,8 @@ rte_mempool_generic_put(struct rte_mempool *mp, void * const *obj_table,
unsigned int n, struct rte_mempool_cache *cache)
{
rte_mempool_trace_generic_put(mp, obj_table, n, cache);
__mempool_check_cookies(mp, obj_table, n, 0);
__mempool_generic_put(mp, obj_table, n, cache);
RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 0);
rte_mempool_do_generic_put(mp, obj_table, n, cache);
}
/**
@ -1437,8 +1438,8 @@ rte_mempool_put(struct rte_mempool *mp, void *obj)
* - <0: Error; code of ring dequeue function.
*/
static __rte_always_inline int
__mempool_generic_get(struct rte_mempool *mp, void **obj_table,
unsigned int n, struct rte_mempool_cache *cache)
rte_mempool_do_generic_get(struct rte_mempool *mp, void **obj_table,
unsigned int n, struct rte_mempool_cache *cache)
{
int ret;
uint32_t index, len;
@ -1477,8 +1478,8 @@ __mempool_generic_get(struct rte_mempool *mp, void **obj_table,
cache->len -= n;
__MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
__MEMPOOL_STAT_ADD(mp, get_success_objs, n);
RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
RTE_MEMPOOL_STAT_ADD(mp, get_success_objs, n);
return 0;
@ -1488,11 +1489,11 @@ ring_dequeue:
ret = rte_mempool_ops_dequeue_bulk(mp, obj_table, n);
if (ret < 0) {
__MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
__MEMPOOL_STAT_ADD(mp, get_fail_objs, n);
RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
RTE_MEMPOOL_STAT_ADD(mp, get_fail_objs, n);
} else {
__MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
__MEMPOOL_STAT_ADD(mp, get_success_objs, n);
RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
RTE_MEMPOOL_STAT_ADD(mp, get_success_objs, n);
}
return ret;
@ -1523,9 +1524,9 @@ rte_mempool_generic_get(struct rte_mempool *mp, void **obj_table,
unsigned int n, struct rte_mempool_cache *cache)
{
int ret;
ret = __mempool_generic_get(mp, obj_table, n, cache);
ret = rte_mempool_do_generic_get(mp, obj_table, n, cache);
if (ret == 0)
__mempool_check_cookies(mp, obj_table, n, 1);
RTE_MEMPOOL_CHECK_COOKIES(mp, obj_table, n, 1);
rte_mempool_trace_generic_get(mp, obj_table, n, cache);
return ret;
}
@ -1616,13 +1617,13 @@ rte_mempool_get_contig_blocks(struct rte_mempool *mp,
ret = rte_mempool_ops_dequeue_contig_blocks(mp, first_obj_table, n);
if (ret == 0) {
__MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
__MEMPOOL_STAT_ADD(mp, get_success_blks, n);
__mempool_contig_blocks_check_cookies(mp, first_obj_table, n,
1);
RTE_MEMPOOL_STAT_ADD(mp, get_success_bulk, 1);
RTE_MEMPOOL_STAT_ADD(mp, get_success_blks, n);
RTE_MEMPOOL_CONTIG_BLOCKS_CHECK_COOKIES(mp, first_obj_table, n,
1);
} else {
__MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
__MEMPOOL_STAT_ADD(mp, get_fail_blks, n);
RTE_MEMPOOL_STAT_ADD(mp, get_fail_bulk, 1);
RTE_MEMPOOL_STAT_ADD(mp, get_fail_blks, n);
}
rte_mempool_trace_get_contig_blocks(mp, first_obj_table, n);