2017-08-24 21:20:12 +00:00
|
|
|
/*-
|
|
|
|
* BSD LICENSE
|
|
|
|
*
|
|
|
|
* Copyright (c) Intel Corporation.
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
*
|
|
|
|
* * Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* * Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in
|
|
|
|
* the documentation and/or other materials provided with the
|
|
|
|
* distribution.
|
|
|
|
* * Neither the name of Intel Corporation nor the names of its
|
|
|
|
* contributors may be used to endorse or promote products derived
|
|
|
|
* from this software without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
|
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "spdk/stdinc.h"
|
|
|
|
|
|
|
|
#include "env_internal.h"
|
|
|
|
|
|
|
|
#include <rte_config.h>
|
|
|
|
#include <rte_eal_memconfig.h>
|
|
|
|
|
|
|
|
#include "spdk_internal/assert.h"
|
|
|
|
|
|
|
|
#include "spdk/assert.h"
|
|
|
|
#include "spdk/likely.h"
|
|
|
|
#include "spdk/queue.h"
|
|
|
|
#include "spdk/util.h"
|
|
|
|
|
|
|
|
#if DEBUG
|
|
|
|
#define DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__)
|
|
|
|
#else
|
|
|
|
#define DEBUG_PRINT(...)
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#define FN_2MB_TO_4KB(fn) (fn << (SHIFT_2MB - SHIFT_4KB))
|
|
|
|
#define FN_4KB_TO_2MB(fn) (fn >> (SHIFT_2MB - SHIFT_4KB))
|
|
|
|
|
2018-01-25 03:36:50 +00:00
|
|
|
#define MAP_256TB_IDX(vfn_2mb) ((vfn_2mb) >> (SHIFT_1GB - SHIFT_2MB))
|
2018-07-05 18:03:42 +00:00
|
|
|
#define MAP_1GB_IDX(vfn_2mb) ((vfn_2mb) & ((1ULL << (SHIFT_1GB - SHIFT_2MB)) - 1))
|
2017-08-24 21:20:12 +00:00
|
|
|
|
2018-11-13 16:43:46 +00:00
|
|
|
#define _2MB_OFFSET(ptr) (((uintptr_t)(ptr)) & (VALUE_2MB - 1))
|
|
|
|
|
2018-09-19 11:36:22 +00:00
|
|
|
/* Page is registered */
|
|
|
|
#define REG_MAP_REGISTERED (1ULL << 62)
|
2018-09-12 15:20:38 +00:00
|
|
|
|
|
|
|
/* A notification region barrier. The 2MB translation entry that's marked
|
|
|
|
* with this flag must be unregistered separately. This allows contiguous
|
|
|
|
* regions to be unregistered in the same chunks they were registered.
|
|
|
|
*/
|
|
|
|
#define REG_MAP_NOTIFY_START (1ULL << 63)
|
|
|
|
|
2017-08-24 21:20:12 +00:00
|
|
|
/* Translation of a single 2MB page. */
|
|
|
|
struct map_2mb {
|
|
|
|
uint64_t translation_2mb;
|
|
|
|
};
|
|
|
|
|
|
|
|
/* Second-level map table indexed by bits [21..29] of the virtual address.
|
|
|
|
* Each entry contains the address translation or error for entries that haven't
|
|
|
|
* been retrieved yet.
|
|
|
|
*/
|
|
|
|
struct map_1gb {
|
2018-07-05 18:03:42 +00:00
|
|
|
struct map_2mb map[1ULL << (SHIFT_1GB - SHIFT_2MB)];
|
2017-08-24 21:20:12 +00:00
|
|
|
};
|
|
|
|
|
2018-09-18 22:53:25 +00:00
|
|
|
/* Top-level map table indexed by bits [30..47] of the virtual address.
|
2017-08-24 21:20:12 +00:00
|
|
|
* Each entry points to a second-level map table or NULL.
|
|
|
|
*/
|
2018-01-25 03:36:50 +00:00
|
|
|
struct map_256tb {
|
2018-07-05 18:03:42 +00:00
|
|
|
struct map_1gb *map[1ULL << (SHIFT_256TB - SHIFT_1GB)];
|
2017-08-24 21:20:12 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
/* Page-granularity memory address translation */
|
|
|
|
struct spdk_mem_map {
|
2018-01-25 03:36:50 +00:00
|
|
|
struct map_256tb map_256tb;
|
2017-08-24 21:20:12 +00:00
|
|
|
pthread_mutex_t mutex;
|
|
|
|
uint64_t default_translation;
|
2018-09-11 21:32:54 +00:00
|
|
|
struct spdk_mem_map_ops ops;
|
2017-08-24 21:20:12 +00:00
|
|
|
void *cb_ctx;
|
|
|
|
TAILQ_ENTRY(spdk_mem_map) tailq;
|
|
|
|
};
|
|
|
|
|
2018-09-12 15:20:38 +00:00
|
|
|
/* Registrations map. The 64 bit translations are bit fields with the
|
|
|
|
* following layout (starting with the low bits):
|
2018-09-19 11:36:22 +00:00
|
|
|
* 0 - 61 : reserved
|
|
|
|
* 62 - 63 : flags
|
2018-09-12 15:20:38 +00:00
|
|
|
*/
|
2017-08-24 21:20:12 +00:00
|
|
|
static struct spdk_mem_map *g_mem_reg_map;
|
|
|
|
static TAILQ_HEAD(, spdk_mem_map) g_spdk_mem_maps = TAILQ_HEAD_INITIALIZER(g_spdk_mem_maps);
|
|
|
|
static pthread_mutex_t g_spdk_mem_map_mutex = PTHREAD_MUTEX_INITIALIZER;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Walk the currently registered memory via the main memory registration map
|
|
|
|
* and call the new map's notify callback for each virtually contiguous region.
|
|
|
|
*/
|
2018-10-04 07:46:25 +00:00
|
|
|
static int
|
2017-08-24 21:20:12 +00:00
|
|
|
spdk_mem_map_notify_walk(struct spdk_mem_map *map, enum spdk_mem_map_notify_action action)
|
|
|
|
{
|
2018-01-25 03:36:50 +00:00
|
|
|
size_t idx_256tb;
|
2018-10-04 07:46:25 +00:00
|
|
|
uint64_t idx_1gb;
|
2018-10-10 14:35:08 +00:00
|
|
|
uint64_t contig_start = UINT64_MAX;
|
|
|
|
uint64_t contig_end = UINT64_MAX;
|
2018-10-04 07:46:25 +00:00
|
|
|
struct map_1gb *map_1gb;
|
|
|
|
int rc;
|
2017-08-24 21:20:12 +00:00
|
|
|
|
|
|
|
if (!g_mem_reg_map) {
|
2018-10-04 07:46:25 +00:00
|
|
|
return -EINVAL;
|
2017-08-24 21:20:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Hold the memory registration map mutex so no new registrations can be added while we are looping. */
|
|
|
|
pthread_mutex_lock(&g_mem_reg_map->mutex);
|
|
|
|
|
2018-01-25 03:36:50 +00:00
|
|
|
for (idx_256tb = 0;
|
|
|
|
idx_256tb < sizeof(g_mem_reg_map->map_256tb.map) / sizeof(g_mem_reg_map->map_256tb.map[0]);
|
|
|
|
idx_256tb++) {
|
2018-10-04 07:46:25 +00:00
|
|
|
map_1gb = g_mem_reg_map->map_256tb.map[idx_256tb];
|
2017-08-24 21:20:12 +00:00
|
|
|
|
|
|
|
if (!map_1gb) {
|
2018-10-10 14:35:08 +00:00
|
|
|
if (contig_start != UINT64_MAX) {
|
2018-10-04 07:46:25 +00:00
|
|
|
/* End of of a virtually contiguous range */
|
|
|
|
rc = map->ops.notify_cb(map->cb_ctx, map, action,
|
|
|
|
(void *)contig_start,
|
|
|
|
contig_end - contig_start + VALUE_2MB);
|
|
|
|
/* Don't bother handling unregister failures. It can't be any worse */
|
|
|
|
if (rc != 0 && action == SPDK_MEM_MAP_NOTIFY_REGISTER) {
|
|
|
|
goto err_unregister;
|
|
|
|
}
|
|
|
|
}
|
2018-10-10 14:35:08 +00:00
|
|
|
contig_start = UINT64_MAX;
|
2017-08-24 21:20:12 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (idx_1gb = 0; idx_1gb < sizeof(map_1gb->map) / sizeof(map_1gb->map[0]); idx_1gb++) {
|
2018-09-19 11:36:22 +00:00
|
|
|
if ((map_1gb->map[idx_1gb].translation_2mb & REG_MAP_REGISTERED) &&
|
2018-09-12 15:20:38 +00:00
|
|
|
(contig_start == UINT64_MAX ||
|
|
|
|
(map_1gb->map[idx_1gb].translation_2mb & REG_MAP_NOTIFY_START) == 0)) {
|
2017-08-24 21:20:12 +00:00
|
|
|
/* Rebuild the virtual address from the indexes */
|
2018-01-25 03:36:50 +00:00
|
|
|
uint64_t vaddr = (idx_256tb << SHIFT_1GB) | (idx_1gb << SHIFT_2MB);
|
2017-08-24 21:20:12 +00:00
|
|
|
|
2018-10-10 14:35:08 +00:00
|
|
|
if (contig_start == UINT64_MAX) {
|
2017-08-24 21:20:12 +00:00
|
|
|
contig_start = vaddr;
|
|
|
|
}
|
2018-10-04 07:46:25 +00:00
|
|
|
|
2017-08-24 21:20:12 +00:00
|
|
|
contig_end = vaddr;
|
|
|
|
} else {
|
2018-10-10 14:35:08 +00:00
|
|
|
if (contig_start != UINT64_MAX) {
|
2018-10-04 07:46:25 +00:00
|
|
|
/* End of of a virtually contiguous range */
|
|
|
|
rc = map->ops.notify_cb(map->cb_ctx, map, action,
|
|
|
|
(void *)contig_start,
|
|
|
|
contig_end - contig_start + VALUE_2MB);
|
|
|
|
/* Don't bother handling unregister failures. It can't be any worse */
|
|
|
|
if (rc != 0 && action == SPDK_MEM_MAP_NOTIFY_REGISTER) {
|
|
|
|
goto err_unregister;
|
|
|
|
}
|
2018-09-12 15:20:38 +00:00
|
|
|
|
|
|
|
/* This page might be a part of a neighbour region, so process
|
|
|
|
* it again. The idx_1gb will be incremented immediately.
|
|
|
|
*/
|
|
|
|
idx_1gb--;
|
2018-10-04 07:46:25 +00:00
|
|
|
}
|
2018-10-10 14:35:08 +00:00
|
|
|
contig_start = UINT64_MAX;
|
2018-10-04 07:46:25 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pthread_mutex_unlock(&g_mem_reg_map->mutex);
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
err_unregister:
|
|
|
|
/* Unwind to the first empty translation so we don't unregister
|
|
|
|
* a region that just failed to register.
|
|
|
|
*/
|
|
|
|
idx_256tb = MAP_256TB_IDX((contig_start >> SHIFT_2MB) - 1);
|
|
|
|
idx_1gb = MAP_1GB_IDX((contig_start >> SHIFT_2MB) - 1);
|
2018-10-10 14:35:08 +00:00
|
|
|
contig_start = UINT64_MAX;
|
|
|
|
contig_end = UINT64_MAX;
|
2018-10-04 07:46:25 +00:00
|
|
|
|
|
|
|
/* Unregister any memory we managed to register before the failure */
|
|
|
|
for (; idx_256tb < SIZE_MAX; idx_256tb--) {
|
|
|
|
map_1gb = g_mem_reg_map->map_256tb.map[idx_256tb];
|
|
|
|
|
|
|
|
if (!map_1gb) {
|
2018-10-10 14:35:08 +00:00
|
|
|
if (contig_end != UINT64_MAX) {
|
2018-10-04 07:46:25 +00:00
|
|
|
/* End of of a virtually contiguous range */
|
|
|
|
map->ops.notify_cb(map->cb_ctx, map,
|
|
|
|
SPDK_MEM_MAP_NOTIFY_UNREGISTER,
|
|
|
|
(void *)contig_start,
|
|
|
|
contig_end - contig_start + VALUE_2MB);
|
|
|
|
}
|
2018-10-10 14:35:08 +00:00
|
|
|
contig_end = UINT64_MAX;
|
2018-10-04 07:46:25 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (; idx_1gb < UINT64_MAX; idx_1gb--) {
|
2018-09-19 11:36:22 +00:00
|
|
|
if ((map_1gb->map[idx_1gb].translation_2mb & REG_MAP_REGISTERED) &&
|
2018-09-12 15:20:38 +00:00
|
|
|
(contig_end == UINT64_MAX || (map_1gb->map[idx_1gb].translation_2mb & REG_MAP_NOTIFY_START) == 0)) {
|
2018-10-04 07:46:25 +00:00
|
|
|
/* Rebuild the virtual address from the indexes */
|
|
|
|
uint64_t vaddr = (idx_256tb << SHIFT_1GB) | (idx_1gb << SHIFT_2MB);
|
|
|
|
|
2018-10-10 14:35:08 +00:00
|
|
|
if (contig_end == UINT64_MAX) {
|
2018-10-04 07:46:25 +00:00
|
|
|
contig_end = vaddr;
|
|
|
|
}
|
|
|
|
contig_start = vaddr;
|
|
|
|
} else {
|
2018-10-10 14:35:08 +00:00
|
|
|
if (contig_end != UINT64_MAX) {
|
2018-10-04 07:46:25 +00:00
|
|
|
/* End of of a virtually contiguous range */
|
|
|
|
map->ops.notify_cb(map->cb_ctx, map,
|
|
|
|
SPDK_MEM_MAP_NOTIFY_UNREGISTER,
|
|
|
|
(void *)contig_start,
|
|
|
|
contig_end - contig_start + VALUE_2MB);
|
2018-09-12 15:20:38 +00:00
|
|
|
idx_1gb++;
|
2018-10-04 07:46:25 +00:00
|
|
|
}
|
2018-10-10 14:35:08 +00:00
|
|
|
contig_end = UINT64_MAX;
|
2017-08-24 21:20:12 +00:00
|
|
|
}
|
|
|
|
}
|
2018-10-04 07:46:25 +00:00
|
|
|
idx_1gb = sizeof(map_1gb->map) / sizeof(map_1gb->map[0]) - 1;
|
2017-08-24 21:20:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
pthread_mutex_unlock(&g_mem_reg_map->mutex);
|
2018-10-04 07:46:25 +00:00
|
|
|
return rc;
|
2017-08-24 21:20:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
struct spdk_mem_map *
|
2018-09-11 22:03:31 +00:00
|
|
|
spdk_mem_map_alloc(uint64_t default_translation, const struct spdk_mem_map_ops *ops, void *cb_ctx)
|
2017-08-24 21:20:12 +00:00
|
|
|
{
|
|
|
|
struct spdk_mem_map *map;
|
2018-10-04 07:46:25 +00:00
|
|
|
int rc;
|
2017-08-24 21:20:12 +00:00
|
|
|
|
|
|
|
map = calloc(1, sizeof(*map));
|
|
|
|
if (map == NULL) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (pthread_mutex_init(&map->mutex, NULL)) {
|
|
|
|
free(map);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
map->default_translation = default_translation;
|
|
|
|
map->cb_ctx = cb_ctx;
|
2018-09-11 22:03:31 +00:00
|
|
|
if (ops) {
|
|
|
|
map->ops = *ops;
|
|
|
|
}
|
2017-08-24 21:20:12 +00:00
|
|
|
|
2018-09-11 22:03:31 +00:00
|
|
|
if (ops && ops->notify_cb) {
|
2018-10-11 07:49:41 +00:00
|
|
|
pthread_mutex_lock(&g_spdk_mem_map_mutex);
|
2018-10-04 07:46:25 +00:00
|
|
|
rc = spdk_mem_map_notify_walk(map, SPDK_MEM_MAP_NOTIFY_REGISTER);
|
|
|
|
if (rc != 0) {
|
|
|
|
pthread_mutex_unlock(&g_spdk_mem_map_mutex);
|
|
|
|
DEBUG_PRINT("Initial mem_map notify failed\n");
|
|
|
|
pthread_mutex_destroy(&map->mutex);
|
|
|
|
free(map);
|
|
|
|
return NULL;
|
|
|
|
}
|
2017-08-24 21:20:12 +00:00
|
|
|
TAILQ_INSERT_TAIL(&g_spdk_mem_maps, map, tailq);
|
2018-10-11 07:49:41 +00:00
|
|
|
pthread_mutex_unlock(&g_spdk_mem_map_mutex);
|
2017-08-24 21:20:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return map;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
spdk_mem_map_free(struct spdk_mem_map **pmap)
|
|
|
|
{
|
|
|
|
struct spdk_mem_map *map;
|
|
|
|
size_t i;
|
|
|
|
|
|
|
|
if (!pmap) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
map = *pmap;
|
|
|
|
|
|
|
|
if (!map) {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2018-10-10 17:29:04 +00:00
|
|
|
if (map->ops.notify_cb) {
|
|
|
|
pthread_mutex_lock(&g_spdk_mem_map_mutex);
|
|
|
|
spdk_mem_map_notify_walk(map, SPDK_MEM_MAP_NOTIFY_UNREGISTER);
|
|
|
|
TAILQ_REMOVE(&g_spdk_mem_maps, map, tailq);
|
|
|
|
pthread_mutex_unlock(&g_spdk_mem_map_mutex);
|
|
|
|
}
|
2017-08-24 21:20:12 +00:00
|
|
|
|
2018-01-25 03:36:50 +00:00
|
|
|
for (i = 0; i < sizeof(map->map_256tb.map) / sizeof(map->map_256tb.map[0]); i++) {
|
|
|
|
free(map->map_256tb.map[i]);
|
2017-08-24 21:20:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
pthread_mutex_destroy(&map->mutex);
|
|
|
|
|
|
|
|
free(map);
|
|
|
|
*pmap = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
spdk_mem_register(void *vaddr, size_t len)
|
|
|
|
{
|
|
|
|
struct spdk_mem_map *map;
|
|
|
|
int rc;
|
|
|
|
void *seg_vaddr;
|
|
|
|
size_t seg_len;
|
2018-09-19 11:36:22 +00:00
|
|
|
uint64_t reg;
|
2017-08-24 21:20:12 +00:00
|
|
|
|
2018-01-25 03:36:50 +00:00
|
|
|
if ((uintptr_t)vaddr & ~MASK_256TB) {
|
2017-08-24 21:20:12 +00:00
|
|
|
DEBUG_PRINT("invalid usermode virtual address %p\n", vaddr);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (((uintptr_t)vaddr & MASK_2MB) || (len & MASK_2MB)) {
|
|
|
|
DEBUG_PRINT("invalid %s parameters, vaddr=%p len=%ju\n",
|
|
|
|
__func__, vaddr, len);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2018-09-19 11:36:22 +00:00
|
|
|
if (len == 0) {
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-08-24 21:20:12 +00:00
|
|
|
pthread_mutex_lock(&g_spdk_mem_map_mutex);
|
|
|
|
|
2018-09-19 11:36:22 +00:00
|
|
|
seg_vaddr = vaddr;
|
|
|
|
seg_len = len;
|
|
|
|
while (seg_len > 0) {
|
|
|
|
reg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)seg_vaddr, NULL);
|
|
|
|
if (reg & REG_MAP_REGISTERED) {
|
|
|
|
pthread_mutex_unlock(&g_spdk_mem_map_mutex);
|
|
|
|
return -EBUSY;
|
|
|
|
}
|
|
|
|
seg_vaddr += VALUE_2MB;
|
|
|
|
seg_len -= VALUE_2MB;
|
|
|
|
}
|
|
|
|
|
2017-08-24 21:20:12 +00:00
|
|
|
seg_vaddr = vaddr;
|
|
|
|
seg_len = 0;
|
|
|
|
while (len > 0) {
|
2018-09-12 15:20:38 +00:00
|
|
|
spdk_mem_map_set_translation(g_mem_reg_map, (uint64_t)vaddr, VALUE_2MB,
|
2018-09-19 11:36:22 +00:00
|
|
|
seg_len == 0 ? REG_MAP_REGISTERED | REG_MAP_NOTIFY_START : REG_MAP_REGISTERED);
|
|
|
|
seg_len += VALUE_2MB;
|
2017-08-24 21:20:12 +00:00
|
|
|
vaddr += VALUE_2MB;
|
|
|
|
len -= VALUE_2MB;
|
|
|
|
}
|
|
|
|
|
2018-09-19 11:36:22 +00:00
|
|
|
TAILQ_FOREACH(map, &g_spdk_mem_maps, tailq) {
|
|
|
|
rc = map->ops.notify_cb(map->cb_ctx, map, SPDK_MEM_MAP_NOTIFY_REGISTER, seg_vaddr, seg_len);
|
|
|
|
if (rc != 0) {
|
|
|
|
pthread_mutex_unlock(&g_spdk_mem_map_mutex);
|
|
|
|
return rc;
|
2017-08-24 21:20:12 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pthread_mutex_unlock(&g_spdk_mem_map_mutex);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
spdk_mem_unregister(void *vaddr, size_t len)
|
|
|
|
{
|
|
|
|
struct spdk_mem_map *map;
|
|
|
|
int rc;
|
|
|
|
void *seg_vaddr;
|
|
|
|
size_t seg_len;
|
2018-09-13 10:34:14 +00:00
|
|
|
uint64_t reg, newreg;
|
2017-08-24 21:20:12 +00:00
|
|
|
|
2018-01-25 03:36:50 +00:00
|
|
|
if ((uintptr_t)vaddr & ~MASK_256TB) {
|
2017-08-24 21:20:12 +00:00
|
|
|
DEBUG_PRINT("invalid usermode virtual address %p\n", vaddr);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (((uintptr_t)vaddr & MASK_2MB) || (len & MASK_2MB)) {
|
|
|
|
DEBUG_PRINT("invalid %s parameters, vaddr=%p len=%ju\n",
|
|
|
|
__func__, vaddr, len);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
pthread_mutex_lock(&g_spdk_mem_map_mutex);
|
|
|
|
|
2018-09-13 10:34:14 +00:00
|
|
|
/* The first page must be a start of a region. Also check if it's
|
|
|
|
* registered to make sure we don't return -ERANGE for non-registered
|
|
|
|
* regions.
|
|
|
|
*/
|
|
|
|
reg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)vaddr, NULL);
|
|
|
|
if ((reg & REG_MAP_REGISTERED) && (reg & REG_MAP_NOTIFY_START) == 0) {
|
|
|
|
pthread_mutex_unlock(&g_spdk_mem_map_mutex);
|
|
|
|
return -ERANGE;
|
|
|
|
}
|
|
|
|
|
2017-08-24 21:20:12 +00:00
|
|
|
seg_vaddr = vaddr;
|
|
|
|
seg_len = len;
|
|
|
|
while (seg_len > 0) {
|
2018-09-12 15:20:38 +00:00
|
|
|
reg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)seg_vaddr, NULL);
|
2018-09-19 11:36:22 +00:00
|
|
|
if ((reg & REG_MAP_REGISTERED) == 0) {
|
2017-08-24 21:20:12 +00:00
|
|
|
pthread_mutex_unlock(&g_spdk_mem_map_mutex);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
seg_vaddr += VALUE_2MB;
|
|
|
|
seg_len -= VALUE_2MB;
|
|
|
|
}
|
|
|
|
|
2018-09-13 10:34:14 +00:00
|
|
|
newreg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)seg_vaddr, NULL);
|
|
|
|
/* If the next page is registered, it must be a start of a region as well,
|
|
|
|
* otherwise we'd be unregistering only a part of a region.
|
|
|
|
*/
|
|
|
|
if ((newreg & REG_MAP_NOTIFY_START) == 0 && (newreg & REG_MAP_REGISTERED)) {
|
|
|
|
pthread_mutex_unlock(&g_spdk_mem_map_mutex);
|
|
|
|
return -ERANGE;
|
|
|
|
}
|
2017-08-24 21:20:12 +00:00
|
|
|
seg_vaddr = vaddr;
|
|
|
|
seg_len = 0;
|
2018-09-13 10:34:14 +00:00
|
|
|
|
2017-08-24 21:20:12 +00:00
|
|
|
while (len > 0) {
|
2018-09-12 15:20:38 +00:00
|
|
|
reg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)vaddr, NULL);
|
2018-09-19 11:36:22 +00:00
|
|
|
spdk_mem_map_set_translation(g_mem_reg_map, (uint64_t)vaddr, VALUE_2MB, 0);
|
|
|
|
|
|
|
|
if (seg_len > 0 && (reg & REG_MAP_NOTIFY_START)) {
|
|
|
|
TAILQ_FOREACH(map, &g_spdk_mem_maps, tailq) {
|
|
|
|
rc = map->ops.notify_cb(map->cb_ctx, map, SPDK_MEM_MAP_NOTIFY_UNREGISTER, seg_vaddr, seg_len);
|
|
|
|
if (rc != 0) {
|
|
|
|
pthread_mutex_unlock(&g_spdk_mem_map_mutex);
|
|
|
|
return rc;
|
2017-08-24 21:20:12 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-10 15:23:24 +00:00
|
|
|
seg_vaddr = vaddr;
|
|
|
|
seg_len = VALUE_2MB;
|
2017-08-24 21:20:12 +00:00
|
|
|
} else {
|
|
|
|
seg_len += VALUE_2MB;
|
|
|
|
}
|
|
|
|
|
|
|
|
vaddr += VALUE_2MB;
|
|
|
|
len -= VALUE_2MB;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (seg_len > 0) {
|
|
|
|
TAILQ_FOREACH(map, &g_spdk_mem_maps, tailq) {
|
2018-09-11 21:32:54 +00:00
|
|
|
rc = map->ops.notify_cb(map->cb_ctx, map, SPDK_MEM_MAP_NOTIFY_UNREGISTER, seg_vaddr, seg_len);
|
2017-08-24 21:20:12 +00:00
|
|
|
if (rc != 0) {
|
|
|
|
pthread_mutex_unlock(&g_spdk_mem_map_mutex);
|
|
|
|
return rc;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pthread_mutex_unlock(&g_spdk_mem_map_mutex);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct map_1gb *
|
|
|
|
spdk_mem_map_get_map_1gb(struct spdk_mem_map *map, uint64_t vfn_2mb)
|
|
|
|
{
|
|
|
|
struct map_1gb *map_1gb;
|
2018-01-25 03:36:50 +00:00
|
|
|
uint64_t idx_256tb = MAP_256TB_IDX(vfn_2mb);
|
2017-08-24 21:20:12 +00:00
|
|
|
size_t i;
|
|
|
|
|
2018-07-05 18:23:55 +00:00
|
|
|
if (spdk_unlikely(idx_256tb >= SPDK_COUNTOF(map->map_256tb.map))) {
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2018-01-25 03:36:50 +00:00
|
|
|
map_1gb = map->map_256tb.map[idx_256tb];
|
2017-08-24 21:20:12 +00:00
|
|
|
|
|
|
|
if (!map_1gb) {
|
|
|
|
pthread_mutex_lock(&map->mutex);
|
|
|
|
|
|
|
|
/* Recheck to make sure nobody else got the mutex first. */
|
2018-01-25 03:36:50 +00:00
|
|
|
map_1gb = map->map_256tb.map[idx_256tb];
|
2017-08-24 21:20:12 +00:00
|
|
|
if (!map_1gb) {
|
|
|
|
map_1gb = malloc(sizeof(struct map_1gb));
|
|
|
|
if (map_1gb) {
|
|
|
|
/* initialize all entries to default translation */
|
|
|
|
for (i = 0; i < SPDK_COUNTOF(map_1gb->map); i++) {
|
|
|
|
map_1gb->map[i].translation_2mb = map->default_translation;
|
|
|
|
}
|
2018-01-25 03:36:50 +00:00
|
|
|
map->map_256tb.map[idx_256tb] = map_1gb;
|
2017-08-24 21:20:12 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pthread_mutex_unlock(&map->mutex);
|
|
|
|
|
|
|
|
if (!map_1gb) {
|
|
|
|
DEBUG_PRINT("allocation failed\n");
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return map_1gb;
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
spdk_mem_map_set_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size,
|
|
|
|
uint64_t translation)
|
|
|
|
{
|
|
|
|
uint64_t vfn_2mb;
|
|
|
|
struct map_1gb *map_1gb;
|
|
|
|
uint64_t idx_1gb;
|
|
|
|
struct map_2mb *map_2mb;
|
|
|
|
|
2018-01-25 03:36:50 +00:00
|
|
|
if ((uintptr_t)vaddr & ~MASK_256TB) {
|
2017-08-24 21:20:12 +00:00
|
|
|
DEBUG_PRINT("invalid usermode virtual address %lu\n", vaddr);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2018-07-05 11:46:23 +00:00
|
|
|
/* For now, only 2 MB-aligned registrations are supported */
|
2017-08-24 21:20:12 +00:00
|
|
|
if (((uintptr_t)vaddr & MASK_2MB) || (size & MASK_2MB)) {
|
|
|
|
DEBUG_PRINT("invalid %s parameters, vaddr=%lu len=%ju\n",
|
|
|
|
__func__, vaddr, size);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
vfn_2mb = vaddr >> SHIFT_2MB;
|
|
|
|
|
|
|
|
while (size) {
|
|
|
|
map_1gb = spdk_mem_map_get_map_1gb(map, vfn_2mb);
|
|
|
|
if (!map_1gb) {
|
|
|
|
DEBUG_PRINT("could not get %p map\n", (void *)vaddr);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
idx_1gb = MAP_1GB_IDX(vfn_2mb);
|
|
|
|
map_2mb = &map_1gb->map[idx_1gb];
|
|
|
|
map_2mb->translation_2mb = translation;
|
|
|
|
|
|
|
|
size -= VALUE_2MB;
|
|
|
|
vfn_2mb++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
spdk_mem_map_clear_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size)
|
|
|
|
{
|
|
|
|
uint64_t vfn_2mb;
|
|
|
|
struct map_1gb *map_1gb;
|
|
|
|
uint64_t idx_1gb;
|
|
|
|
struct map_2mb *map_2mb;
|
|
|
|
|
2018-01-25 03:36:50 +00:00
|
|
|
if ((uintptr_t)vaddr & ~MASK_256TB) {
|
2017-08-24 21:20:12 +00:00
|
|
|
DEBUG_PRINT("invalid usermode virtual address %lu\n", vaddr);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2018-07-05 11:46:23 +00:00
|
|
|
/* For now, only 2 MB-aligned registrations are supported */
|
2017-08-24 21:20:12 +00:00
|
|
|
if (((uintptr_t)vaddr & MASK_2MB) || (size & MASK_2MB)) {
|
|
|
|
DEBUG_PRINT("invalid %s parameters, vaddr=%lu len=%ju\n",
|
|
|
|
__func__, vaddr, size);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
vfn_2mb = vaddr >> SHIFT_2MB;
|
|
|
|
|
|
|
|
while (size) {
|
|
|
|
map_1gb = spdk_mem_map_get_map_1gb(map, vfn_2mb);
|
|
|
|
if (!map_1gb) {
|
|
|
|
DEBUG_PRINT("could not get %p map\n", (void *)vaddr);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
idx_1gb = MAP_1GB_IDX(vfn_2mb);
|
|
|
|
map_2mb = &map_1gb->map[idx_1gb];
|
2017-08-24 21:43:34 +00:00
|
|
|
map_2mb->translation_2mb = map->default_translation;
|
2017-08-24 21:20:12 +00:00
|
|
|
|
|
|
|
size -= VALUE_2MB;
|
|
|
|
vfn_2mb++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
uint64_t
|
2018-09-06 20:07:42 +00:00
|
|
|
spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, uint64_t *size)
|
2017-08-24 21:20:12 +00:00
|
|
|
{
|
|
|
|
const struct map_1gb *map_1gb;
|
|
|
|
const struct map_2mb *map_2mb;
|
2018-01-25 03:36:50 +00:00
|
|
|
uint64_t idx_256tb;
|
2017-08-24 21:20:12 +00:00
|
|
|
uint64_t idx_1gb;
|
|
|
|
uint64_t vfn_2mb;
|
2018-09-11 23:05:50 +00:00
|
|
|
uint64_t cur_size;
|
|
|
|
uint64_t prev_translation;
|
2018-12-07 22:33:38 +00:00
|
|
|
uint64_t orig_translation;
|
2017-08-24 21:20:12 +00:00
|
|
|
|
2018-01-25 03:36:50 +00:00
|
|
|
if (spdk_unlikely(vaddr & ~MASK_256TB)) {
|
2017-08-24 21:20:12 +00:00
|
|
|
DEBUG_PRINT("invalid usermode virtual address %p\n", (void *)vaddr);
|
|
|
|
return map->default_translation;
|
|
|
|
}
|
|
|
|
|
|
|
|
vfn_2mb = vaddr >> SHIFT_2MB;
|
2018-01-25 03:36:50 +00:00
|
|
|
idx_256tb = MAP_256TB_IDX(vfn_2mb);
|
2017-08-24 21:20:12 +00:00
|
|
|
idx_1gb = MAP_1GB_IDX(vfn_2mb);
|
|
|
|
|
2018-01-25 03:36:50 +00:00
|
|
|
map_1gb = map->map_256tb.map[idx_256tb];
|
2017-08-24 21:20:12 +00:00
|
|
|
if (spdk_unlikely(!map_1gb)) {
|
|
|
|
return map->default_translation;
|
|
|
|
}
|
|
|
|
|
2018-11-13 16:43:46 +00:00
|
|
|
cur_size = VALUE_2MB - _2MB_OFFSET(vaddr);
|
2018-09-11 23:05:50 +00:00
|
|
|
map_2mb = &map_1gb->map[idx_1gb];
|
|
|
|
if (size == NULL || map->ops.are_contiguous == NULL ||
|
|
|
|
map_2mb->translation_2mb == map->default_translation) {
|
2018-12-28 09:29:42 +00:00
|
|
|
if (size != NULL) {
|
2018-12-28 10:15:29 +00:00
|
|
|
*size = spdk_min(*size, cur_size);
|
2018-12-28 09:29:42 +00:00
|
|
|
}
|
2018-09-11 23:05:50 +00:00
|
|
|
return map_2mb->translation_2mb;
|
|
|
|
}
|
|
|
|
|
2018-12-07 22:33:38 +00:00
|
|
|
orig_translation = map_2mb->translation_2mb;
|
|
|
|
prev_translation = orig_translation;
|
2018-12-28 09:29:42 +00:00
|
|
|
while (cur_size < *size) {
|
2018-09-11 23:05:50 +00:00
|
|
|
vfn_2mb++;
|
|
|
|
idx_256tb = MAP_256TB_IDX(vfn_2mb);
|
|
|
|
idx_1gb = MAP_1GB_IDX(vfn_2mb);
|
|
|
|
|
|
|
|
map_1gb = map->map_256tb.map[idx_256tb];
|
|
|
|
if (spdk_unlikely(!map_1gb)) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
map_2mb = &map_1gb->map[idx_1gb];
|
|
|
|
if (!map->ops.are_contiguous(prev_translation, map_2mb->translation_2mb)) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
cur_size += VALUE_2MB;
|
|
|
|
prev_translation = map_2mb->translation_2mb;
|
|
|
|
}
|
|
|
|
|
2018-12-28 10:15:29 +00:00
|
|
|
*size = spdk_min(*size, cur_size);
|
2018-12-07 22:33:38 +00:00
|
|
|
return orig_translation;
|
2017-08-24 21:20:12 +00:00
|
|
|
}
|
|
|
|
|
2018-05-09 05:28:38 +00:00
|
|
|
#if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0)
|
|
|
|
static void
|
|
|
|
memory_hotplug_cb(enum rte_mem_event event_type,
|
|
|
|
const void *addr, size_t len, void *arg)
|
|
|
|
{
|
|
|
|
if (event_type == RTE_MEM_EVENT_ALLOC) {
|
2018-11-17 12:49:21 +00:00
|
|
|
spdk_mem_register((void *)addr, len);
|
|
|
|
|
|
|
|
/* Now mark each segment so that DPDK won't later free it.
|
|
|
|
* This ensures we don't have to deal with the memory
|
|
|
|
* getting freed in different units than it was allocated.
|
|
|
|
*/
|
2018-10-10 15:52:33 +00:00
|
|
|
while (len > 0) {
|
|
|
|
struct rte_memseg *seg;
|
|
|
|
|
|
|
|
seg = rte_mem_virt2memseg(addr, NULL);
|
|
|
|
assert(seg != NULL);
|
2018-11-17 12:49:21 +00:00
|
|
|
seg->flags |= RTE_MEMSEG_FLAG_DO_NOT_FREE;
|
2018-10-10 15:52:33 +00:00
|
|
|
addr = (void *)((uintptr_t)addr + seg->hugepage_sz);
|
|
|
|
len -= seg->hugepage_sz;
|
|
|
|
}
|
2018-05-09 05:28:38 +00:00
|
|
|
} else if (event_type == RTE_MEM_EVENT_FREE) {
|
|
|
|
spdk_mem_unregister((void *)addr, len);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
memory_iter_cb(const struct rte_memseg_list *msl,
|
|
|
|
const struct rte_memseg *ms, size_t len, void *arg)
|
|
|
|
{
|
|
|
|
return spdk_mem_register(ms->addr, len);
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2017-12-18 19:57:01 +00:00
|
|
|
int
|
2017-08-24 21:20:12 +00:00
|
|
|
spdk_mem_map_init(void)
|
|
|
|
{
|
|
|
|
g_mem_reg_map = spdk_mem_map_alloc(0, NULL, NULL);
|
|
|
|
if (g_mem_reg_map == NULL) {
|
|
|
|
DEBUG_PRINT("memory registration map allocation failed\n");
|
2017-12-18 19:57:01 +00:00
|
|
|
return -1;
|
2017-08-24 21:20:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Walk all DPDK memory segments and register them
|
|
|
|
* with the master memory map
|
|
|
|
*/
|
2018-05-09 05:28:38 +00:00
|
|
|
#if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0)
|
|
|
|
rte_mem_event_callback_register("spdk", memory_hotplug_cb, NULL);
|
|
|
|
rte_memseg_contig_walk(memory_iter_cb, NULL);
|
|
|
|
#else
|
|
|
|
struct rte_mem_config *mcfg;
|
|
|
|
size_t seg_idx;
|
2017-08-24 21:20:12 +00:00
|
|
|
|
2018-05-09 05:28:38 +00:00
|
|
|
mcfg = rte_eal_get_configuration()->mem_config;
|
2017-08-24 21:20:12 +00:00
|
|
|
for (seg_idx = 0; seg_idx < RTE_MAX_MEMSEG; seg_idx++) {
|
|
|
|
struct rte_memseg *seg = &mcfg->memseg[seg_idx];
|
|
|
|
|
|
|
|
if (seg->addr == NULL) {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
spdk_mem_register(seg->addr, seg->len);
|
|
|
|
}
|
2018-05-09 05:28:38 +00:00
|
|
|
#endif
|
2017-12-18 19:57:01 +00:00
|
|
|
return 0;
|
2017-08-24 21:20:12 +00:00
|
|
|
}
|