From 6c17f696c10ba64d43ce2712c0ef12f5403bb2dd Mon Sep 17 00:00:00 2001 From: Darek Stojaczyk Date: Mon, 8 Apr 2019 23:39:30 +0200 Subject: [PATCH] vhost: allocate device objects with regular calloc spdk_dma_malloc() is not required here, as the device object is neither DMA-able nor shared between processes. The device structures used to be aligned to cache line size, but that's just a leftover from before sessions were introduced. The device object is just a generic device information that can be accessed from any thread holding the proper mutex. The hot data used in the I/O path sits in the session structure, which is now allocated with posix_memalloc() to ensure proper alignment. Vhost NVMe is an exception, as the device struct is used as hot I/O data for the one and only session it supports, so it's also allocated with posix_memalloc(). While here, also allocate various vhost buffers using spdk_zmalloc() instead of spdk_dma_zmalloc(), as spdk_dma_*malloc() is about to be deprecated. Change-Id: Ic7f63185639b7b98dc1ef756166c826a0af87b44 Signed-off-by: Darek Stojaczyk Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/450551 Tested-by: SPDK CI Jenkins Reviewed-by: Jim Harris Reviewed-by: Ben Walker --- lib/vhost/vhost.c | 11 +++++------ lib/vhost/vhost_blk.c | 8 ++++---- lib/vhost/vhost_nvme.c | 14 +++++++------- lib/vhost/vhost_scsi.c | 11 +++++------ 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/lib/vhost/vhost.c b/lib/vhost/vhost.c index d678d14394..eef28648e3 100644 --- a/lib/vhost/vhost.c +++ b/lib/vhost/vhost.c @@ -1334,14 +1334,13 @@ new_connection(int vid) return -EINVAL; } - vsession = spdk_dma_zmalloc(sizeof(struct spdk_vhost_session) + - vdev->backend->session_ctx_size, - SPDK_CACHE_LINE_SIZE, NULL); - if (vsession == NULL) { - SPDK_ERRLOG("spdk_dma_zmalloc failed\n"); + if (posix_memalign((void **)&vsession, SPDK_CACHE_LINE_SIZE, sizeof(*vsession) + + vdev->backend->session_ctx_size)) { + SPDK_ERRLOG("vsession alloc failed\n"); pthread_mutex_unlock(&g_spdk_vhost_mutex); return -1; } + memset(vsession, 0, sizeof(*vsession) + vdev->backend->session_ctx_size); vsession->vdev = vdev; vsession->id = vdev->vsessions_num++; @@ -1376,7 +1375,7 @@ destroy_connection(int vid) } TAILQ_REMOVE(&vsession->vdev->vsessions, vsession, tailq); - spdk_dma_free(vsession); + free(vsession); pthread_mutex_unlock(&g_spdk_vhost_mutex); } diff --git a/lib/vhost/vhost_blk.c b/lib/vhost/vhost_blk.c index dec3fc854f..5d4b40ec99 100644 --- a/lib/vhost/vhost_blk.c +++ b/lib/vhost/vhost_blk.c @@ -622,7 +622,7 @@ free_task_pool(struct spdk_vhost_blk_session *bvsession) continue; } - spdk_dma_free(vq->tasks); + spdk_free(vq->tasks); vq->tasks = NULL; } } @@ -1008,7 +1008,7 @@ spdk_vhost_blk_construct(const char *name, const char *cpumask, const char *dev_ goto out; } - bvdev = spdk_dma_zmalloc(sizeof(*bvdev), SPDK_CACHE_LINE_SIZE, NULL); + bvdev = calloc(1, sizeof(*bvdev)); if (bvdev == NULL) { ret = -ENOMEM; goto out; @@ -1057,7 +1057,7 @@ spdk_vhost_blk_construct(const char *name, const char *cpumask, const char *dev_ SPDK_INFOLOG(SPDK_LOG_VHOST, "Controller %s: using bdev '%s'\n", name, dev_name); out: if (ret != 0 && bvdev) { - spdk_dma_free(bvdev); + free(bvdev); } spdk_vhost_unlock(); return ret; @@ -1084,7 +1084,7 @@ spdk_vhost_blk_destroy(struct spdk_vhost_dev *vdev) } bvdev->bdev = NULL; - spdk_dma_free(bvdev); + free(bvdev); return 0; } diff --git a/lib/vhost/vhost_nvme.c b/lib/vhost/vhost_nvme.c index 597c044799..b5d495a933 100644 --- a/lib/vhost/vhost_nvme.c +++ b/lib/vhost/vhost_nvme.c @@ -1047,7 +1047,7 @@ free_task_pool(struct spdk_vhost_nvme_dev *nvme) while (!STAILQ_EMPTY(&nvme->free_tasks)) { task = STAILQ_FIRST(&nvme->free_tasks); STAILQ_REMOVE_HEAD(&nvme->free_tasks, stailq); - spdk_dma_free(task); + spdk_free(task); } } @@ -1379,16 +1379,16 @@ spdk_vhost_nvme_ctrlr_identify_update(struct spdk_vhost_nvme_dev *dev) int spdk_vhost_nvme_dev_construct(const char *name, const char *cpumask, uint32_t num_io_queues) { - struct spdk_vhost_nvme_dev *dev = spdk_dma_zmalloc(sizeof(struct spdk_vhost_nvme_dev), - SPDK_CACHE_LINE_SIZE, NULL); + struct spdk_vhost_nvme_dev *dev; int rc; - if (dev == NULL) { + if (posix_memalign((void **)&dev, SPDK_CACHE_LINE_SIZE, sizeof(*dev))) { return -ENOMEM; } + memset(dev, 0, sizeof(*dev)); if (num_io_queues < 1 || num_io_queues > MAX_IO_QUEUES) { - spdk_dma_free(dev); + free(dev); return -EINVAL; } @@ -1397,7 +1397,7 @@ spdk_vhost_nvme_dev_construct(const char *name, const char *cpumask, uint32_t nu &spdk_vhost_nvme_device_backend); if (rc) { - spdk_dma_free(dev); + free(dev); spdk_vhost_unlock(); return rc; } @@ -1438,7 +1438,7 @@ spdk_vhost_nvme_dev_remove(struct spdk_vhost_dev *vdev) return rc; } - spdk_dma_free(nvme); + free(nvme); return 0; } diff --git a/lib/vhost/vhost_scsi.c b/lib/vhost/vhost_scsi.c index fe28d81373..304dc749ee 100644 --- a/lib/vhost/vhost_scsi.c +++ b/lib/vhost/vhost_scsi.c @@ -98,7 +98,7 @@ struct spdk_vhost_scsi_dev { /* The CPU chosen to poll I/O of all active vhost sessions */ int32_t lcore; -} __rte_cache_aligned; +}; /** Context for a SCSI target in a vhost session */ struct spdk_scsi_dev_session_state { @@ -831,8 +831,7 @@ to_scsi_session(struct spdk_vhost_session *vsession) int spdk_vhost_scsi_dev_construct(const char *name, const char *cpumask) { - struct spdk_vhost_scsi_dev *svdev = spdk_dma_zmalloc(sizeof(struct spdk_vhost_scsi_dev), - SPDK_CACHE_LINE_SIZE, NULL); + struct spdk_vhost_scsi_dev *svdev = calloc(1, sizeof(*svdev)); int rc; if (svdev == NULL) { @@ -844,7 +843,7 @@ spdk_vhost_scsi_dev_construct(const char *name, const char *cpumask) &spdk_vhost_scsi_device_backend); if (rc) { - spdk_dma_free(svdev); + free(svdev); } spdk_vhost_unlock(); @@ -881,7 +880,7 @@ spdk_vhost_scsi_dev_remove(struct spdk_vhost_dev *vdev) return rc; } - spdk_dma_free(svdev); + free(svdev); return 0; } @@ -1244,7 +1243,7 @@ free_task_pool(struct spdk_vhost_scsi_session *svsession) continue; } - spdk_dma_free(vq->tasks); + spdk_free(vq->tasks); vq->tasks = NULL; } }