malloc: add NUMA-aware realloc function

Currently, rte_realloc will not respect original allocation's
NUMA node when memory cannot be resized, and there is no
NUMA-aware equivalent of rte_realloc. This patch adds such a function.

The new API will ensure that reallocated memory stays on
requested NUMA node, as well as allow moving allocated memory
to a different NUMA node.

Signed-off-by: Tomasz Jozwiak <tomaszx.jozwiak@intel.com>
Acked-by: Anatoly Burakov <anatoly.burakov@intel.com>

app/test/test_malloc.c

@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2010-2019 Intel Corporation
  */
 
 #include <stdio.h>
@@ -26,6 +26,13 @@
 
 #define N 10000
 
+
+static int
+is_mem_on_socket(int32_t socket);
+
+static int32_t
+addr_to_socket(void *addr);
+
 /*
  * Malloc
  * ======
@@ -542,7 +549,49 @@ test_realloc(void)
 		return -1;
 	}
 	rte_free(ptr12);
-	return 0;
+
+	/* check realloc_socket part */
+	int32_t socket_count = 0, socket_allocated, socket;
+	int ret = -1;
+	size_t size = 1024;
+
+	ptr1 = NULL;
+	for (socket = 0; socket < RTE_MAX_NUMA_NODES; socket++) {
+		if (is_mem_on_socket(socket)) {
+			int j = 2;
+
+			socket_count++;
+			while (j--) {
+				/* j == 1 -> resizing */
+				ptr2 = rte_realloc_socket(ptr1, size,
+							  RTE_CACHE_LINE_SIZE,
+							  socket);
+				if (ptr2 == NULL) {
+					printf("NULL pointer returned from rte_realloc_socket\n");
+					goto end;
+				}
+
+				ptr1 = ptr2;
+				socket_allocated = addr_to_socket(ptr2);
+				if (socket_allocated != socket) {
+					printf("Requested socket (%d) doesn't mach allocated one (%d)\n",
+					       socket, socket_allocated);
+					goto end;
+				}
+				size += RTE_CACHE_LINE_SIZE;
+			}
+		}
+	}
+
+	/* Print warnign if only a single socket, but don't fail the test */
+	if (socket_count < 2)
+		printf("WARNING: realloc_socket test needs memory on multiple sockets!\n");
+
+	ret = 0;
+end:
+	rte_free(ptr1);
+
+	return ret;
 }
 
 static int

lib/librte_eal/common/include/rte_malloc.h

@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2010-2019 Intel Corporation
  */
 
 #ifndef _RTE_MALLOC_H_
@@ -129,7 +129,32 @@ rte_calloc(const char *type, size_t num, size_t size, unsigned align);
  *   - Otherwise, the pointer to the reallocated memory.
  */
 void *
-rte_realloc(void *ptr, size_t size, unsigned align);
+rte_realloc(void *ptr, size_t size, unsigned int align);
+
+/**
+ * Replacement function for realloc(), using huge-page memory. Reserved area
+ * memory is resized, preserving contents. In NUMA systems, the new area
+ * resides on requested NUMA socket.
+ *
+ * @param ptr
+ *   Pointer to already allocated memory
+ * @param size
+ *   Size (in bytes) of new area. If this is 0, memory is freed.
+ * @param align
+ *   If 0, the return is a pointer that is suitably aligned for any kind of
+ *   variable (in the same manner as malloc()).
+ *   Otherwise, the return is a pointer that is a multiple of *align*. In
+ *   this case, it must obviously be a power of two. (Minimum alignment is the
+ *   cacheline size, i.e. 64-bytes)
+ * @param socket
+ *   NUMA socket to allocate memory on.
+ * @return
+ *   - NULL on error. Not enough memory, or invalid arguments (size is 0,
+ *     align is not a power of two).
+ *   - Otherwise, the pointer to the reallocated memory.
+ */
+void * __rte_experimental
+rte_realloc_socket(void *ptr, size_t size, unsigned int align, int socket);
 
 /**
  * This function allocates memory from the huge-page area of memory. The memory

lib/librte_eal/common/rte_malloc.c

@@ -1,5 +1,5 @@
 /* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2010-2014 Intel Corporation
+ * Copyright(c) 2010-2019 Intel Corporation
  */
 
 #include <stdint.h>
@@ -105,13 +105,13 @@ rte_calloc(const char *type, size_t num, size_t size, unsigned align)
 }
 
 /*
- * Resize allocated memory.
+ * Resize allocated memory on specified heap.
  */
 void *
-rte_realloc(void *ptr, size_t size, unsigned align)
+rte_realloc_socket(void *ptr, size_t size, unsigned int align, int socket)
 {
 	if (ptr == NULL)
-		return rte_malloc(NULL, size, align);
+		return rte_malloc_socket(NULL, size, align, socket);
 
 	struct malloc_elem *elem = malloc_elem_from_data(ptr);
 	if (elem == NULL) {
@@ -120,14 +120,21 @@ rte_realloc(void *ptr, size_t size, unsigned align)
 	}
 
 	size = RTE_CACHE_LINE_ROUNDUP(size), align = RTE_CACHE_LINE_ROUNDUP(align);
-	/* check alignment matches first, and if ok, see if we can resize block */
-	if (RTE_PTR_ALIGN(ptr,align) == ptr &&
+
+	/* check requested socket id and alignment matches first, and if ok,
+	 * see if we can resize block
+	 */
+	if ((socket == SOCKET_ID_ANY ||
+	     (unsigned int)socket == elem->heap->socket_id) &&
+			RTE_PTR_ALIGN(ptr, align) == ptr &&
 			malloc_heap_resize(elem, size) == 0)
 		return ptr;
 
-	/* either alignment is off, or we have no room to expand,
-	 * so move data. */
-	void *new_ptr = rte_malloc(NULL, size, align);
+	/* either requested socket id doesn't match, alignment is off
+	 * or we have no room to expand,
+	 * so move the data.
+	 */
+	void *new_ptr = rte_malloc_socket(NULL, size, align, socket);
 	if (new_ptr == NULL)
 		return NULL;
 	const unsigned old_size = elem->size - MALLOC_ELEM_OVERHEAD;
@@ -137,6 +144,15 @@ rte_realloc(void *ptr, size_t size, unsigned align)
 	return new_ptr;
 }
 
+/*
+ * Resize allocated memory.
+ */
+void *
+rte_realloc(void *ptr, size_t size, unsigned int align)
+{
+	return rte_realloc_socket(ptr, size, align, SOCKET_ID_ANY);
+}
+
 int
 rte_malloc_validate(const void *ptr, size_t *size)
 {

lib/librte_eal/rte_eal_version.map

@@ -359,6 +359,7 @@ EXPERIMENTAL {
 	rte_mp_request_async;
 	rte_mp_sendmsg;
 	rte_option_register;
+	rte_realloc_socket;
 	rte_service_lcore_attr_get;
 	rte_service_lcore_attr_reset_all;
 	rte_service_may_be_active;