d/numam-spdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nvmf: remove rx_desc and tx_desc mempools

Browse Source 
The RDMA tx_desc and rx_desc pools were only used at startup; all
descriptors are immediately allocated and put into a queue, and the
mempool functionality was never used at runtime.

Change-Id: I2882274962550191a555c8483b8f7be2854b32ec
Signed-off-by: Daniel Verkamp <daniel.verkamp@intel.com>
This commit is contained in:
Daniel Verkamp 2016-06-23 15:43:41 -07:00
parent e7dce1cfc0
commit 31d2a116c8
3 changed files with 11 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
lib/nvmf/nvmf.c
View File

@ -65,9 +65,6 @@ SPDK_LOG_REGISTER_TRACE_FLAG("nvmf", SPDK_TRACE_NVMF)
*
* SPDK_NVMF_DESC_POOL_SIZE: The total number of RDMA descriptors
* needed for all possible admin and I/O queue requests.
*
* SPDK_NVMF_TX_DESC_POOL_SIZE: The total number of RDMA descriptors
* needed for all possible admin and I/O queue tx requests.
*/
#define SPDK_NVMF_ADMINQ_POOL_SIZE(spdk) (MAX_SUBSYSTEMS * \
(spdk->MaxSessionsPerSubsystem) * \
@ -81,10 +78,6 @@ SPDK_LOG_REGISTER_TRACE_FLAG("nvmf", SPDK_TRACE_NVMF)
#define SPDK_NVMF_DESC_POOL_SIZE(spdk) (SPDK_NVMF_ADMINQ_POOL_SIZE(spdk) + \
SPDK_NVMF_IOQ_POOL_SIZE(spdk))
/* Per each connection, cq depth is double of sq depth. Double the memory accordingly. */
#define SPDK_NVMF_TX_DESC_POOL_SIZE(spdk) ((SPDK_NVMF_ADMINQ_POOL_SIZE(spdk) + \
SPDK_NVMF_IOQ_POOL_SIZE(spdk)) * 2)
#define SPDK_NVMF_MAX_CONNECTIONS(spdk) (MAX_SUBSYSTEMS * \
((spdk)->MaxSessionsPerSubsystem) * \
((spdk)->MaxConnectionsPerSession))
@ -145,36 +138,6 @@ spdk_nvmf_initialize_pools(struct spdk_nvmf_globals *spdk_nvmf)
spdk_nvmf->bb_large_pool,
(SPDK_NVMF_IOQ_POOL_SIZE(spdk_nvmf) * LARGE_BB_MAX_SIZE));
spdk_nvmf->rx_desc_pool =
rte_mempool_create("RDMA RX Desc Pool",
SPDK_NVMF_DESC_POOL_SIZE(spdk_nvmf),
sizeof(struct nvme_qp_rx_desc),
0, 0,
NULL, NULL, NULL, NULL,
SOCKET_ID_ANY, 0);
if (!spdk_nvmf->rx_desc_pool) {
SPDK_ERRLOG("create RX Desc pool failed\n");
return -1;
}
SPDK_TRACELOG(SPDK_TRACE_DEBUG, "RDMA Rx descriptor pool %p, size 0x%lx bytes\n",
spdk_nvmf->rx_desc_pool,
(SPDK_NVMF_DESC_POOL_SIZE(spdk_nvmf) * sizeof(struct nvme_qp_rx_desc)));
spdk_nvmf->tx_desc_pool =
rte_mempool_create("RDMA TX Desc Pool",
SPDK_NVMF_TX_DESC_POOL_SIZE(spdk_nvmf),
sizeof(struct nvme_qp_tx_desc),
0, 0,
NULL, NULL, NULL, NULL,
SOCKET_ID_ANY, 0);
if (!spdk_nvmf->tx_desc_pool) {
SPDK_ERRLOG("create TX Desc pool failed\n");
return -1;
}
SPDK_TRACELOG(SPDK_TRACE_DEBUG, "RDMA Tx descriptor pool %p, size 0x%lx bytes\n",
spdk_nvmf->tx_desc_pool,
(SPDK_NVMF_TX_DESC_POOL_SIZE(spdk_nvmf) * sizeof(struct nvme_qp_tx_desc)));
return 0;
}
@ -196,8 +159,6 @@ spdk_nvmf_check_pools(void)
int rc = 0;
rc += spdk_nvmf_check_pool(spdk_nvmf->nvme_request_pool, SPDK_NVMF_DESC_POOL_SIZE(spdk_nvmf));
rc += spdk_nvmf_check_pool(spdk_nvmf->rx_desc_pool, SPDK_NVMF_DESC_POOL_SIZE(spdk_nvmf));
rc += spdk_nvmf_check_pool(spdk_nvmf->tx_desc_pool, SPDK_NVMF_TX_DESC_POOL_SIZE(spdk_nvmf));
rc += spdk_nvmf_check_pool(spdk_nvmf->bb_small_pool, SPDK_NVMF_ADMINQ_POOL_SIZE(spdk_nvmf));
rc += spdk_nvmf_check_pool(spdk_nvmf->bb_large_pool, SPDK_NVMF_IOQ_POOL_SIZE(spdk_nvmf));

2
lib/nvmf/nvmf_internal.h
View File

@ -113,8 +113,6 @@ struct spdk_nvmf_globals {
int MaxConnectionsPerSession;
int MaxRecvDataSegmentLength;
struct rte_mempool *rx_desc_pool;
struct rte_mempool *tx_desc_pool;
struct rte_mempool *nvme_request_pool;
struct rte_mempool *bb_small_pool;
struct rte_mempool *bb_large_pool;

23
lib/nvmf/rdma.c
View File

@ -42,6 +42,7 @@
#include <rte_debug.h>
#include <rte_cycles.h>
#include <rte_timer.h>
#include <rte_malloc.h>
#include <rte_mempool.h>
#include "conn.h"
@ -186,7 +187,7 @@ free_qp_desc(struct spdk_nvmf_conn *conn)
SPDK_ERRLOG("Unable to de-register rx mr\n");
}
rte_mempool_put(g_nvmf_tgt.rx_desc_pool, (void *)tmp_rx);
rte_free(tmp_rx);
}
STAILQ_FOREACH(tmp_tx, &conn->rdma.qp_tx_desc, link) {
@ -197,7 +198,7 @@ free_qp_desc(struct spdk_nvmf_conn *conn)
SPDK_ERRLOG("Unable to de-register tx mr\n");
}
rte_mempool_put(g_nvmf_tgt.tx_desc_pool, (void *)tmp_tx);
rte_free(tmp_tx);
}
}
@ -910,9 +911,8 @@ alloc_qp_rx_desc(struct spdk_nvmf_conn *conn)
/* Allocate buffer for rx descriptors (RX WQE + Msg Buffer) */
for (i = 0; i < conn->sq_depth; i++) {
rx_desc = NULL;
rc = rte_mempool_get(g_nvmf_tgt.rx_desc_pool, (void **)&rx_desc);
if ((rc < 0) || !rx_desc) {
rx_desc = rte_zmalloc("nvmf_rx_desc", sizeof(*rx_desc), 0);
if (!rx_desc) {
SPDK_ERRLOG("Unable to get rx desc object\n");
goto fail;
}
@ -998,7 +998,7 @@ fail:
}
}
rte_mempool_put(g_nvmf_tgt.rx_desc_pool, (void *)rx_desc);
rte_free(rx_desc);
}
STAILQ_FOREACH(tmp, &conn->rdma.qp_rx_desc, link) {
@ -1020,7 +1020,7 @@ fail:
SPDK_ERRLOG("Unable to de-register rx mr\n");
}
rte_mempool_put(g_nvmf_tgt.rx_desc_pool, (void *)tmp);
rte_free(tmp);
}
return -ENOMEM;
@ -1036,9 +1036,8 @@ alloc_qp_tx_desc(struct spdk_nvmf_conn *conn)
/* Initialize the tx descriptors */
for (i = 0; i < conn->cq_depth; i++) {
tx_desc = NULL;
rc = rte_mempool_get(g_nvmf_tgt.tx_desc_pool, (void **)&tx_desc);
if ((rc < 0) || !tx_desc) {
tx_desc = rte_zmalloc("nvmf_tx_desc", sizeof(*tx_desc), 0);
if (!tx_desc) {
SPDK_ERRLOG("Unable to get tx desc object\n");
goto fail;
}
@ -1079,7 +1078,7 @@ fail:
}
}
rte_mempool_put(g_nvmf_tgt.tx_desc_pool, (void *)tx_desc);
rte_free(tx_desc);
}
STAILQ_FOREACH(tmp, &conn->rdma.qp_tx_desc, link) {
@ -1090,7 +1089,7 @@ fail:
SPDK_ERRLOG("Unable to de-register tx mr\n");
}
rte_mempool_put(g_nvmf_tgt.tx_desc_pool, (void *)tmp);
rte_free(tmp);
}
return -ENOMEM;