d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

memory: add numa-awareness to malloc

Browse Source 
Signed-off-by: Intel
This commit is contained in:
Intel 2012-12-20 00:00:00 +01:00 committed by Thomas Monjalon
parent 916e4f4f4e
commit dec5e73d78
9 changed files with 206 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
config/defconfig_i686-default-linuxapp-gcc
View File

@ -186,7 +186,6 @@ CONFIG_RTE_LIBRTE_TIMER_DEBUG=n
CONFIG_RTE_LIBRTE_MALLOC=y
CONFIG_RTE_LIBRTE_MALLOC_DEBUG=n
CONFIG_RTE_MALLOC_MEMZONE_SIZE=11M
CONFIG_RTE_MALLOC_PER_NUMA_NODE=y
#
# Compile librte_cmdline

1
config/defconfig_i686-default-linuxapp-icc
View File

@ -186,7 +186,6 @@ CONFIG_RTE_LIBRTE_TIMER_DEBUG=n
CONFIG_RTE_LIBRTE_MALLOC=y
CONFIG_RTE_LIBRTE_MALLOC_DEBUG=n
CONFIG_RTE_MALLOC_MEMZONE_SIZE=11M
CONFIG_RTE_MALLOC_PER_NUMA_NODE=y
#
# Compile librte_cmdline

1
config/defconfig_x86_64-default-linuxapp-gcc
View File

@ -186,7 +186,6 @@ CONFIG_RTE_LIBRTE_TIMER_DEBUG=n
CONFIG_RTE_LIBRTE_MALLOC=y
CONFIG_RTE_LIBRTE_MALLOC_DEBUG=n
CONFIG_RTE_MALLOC_MEMZONE_SIZE=11M
CONFIG_RTE_MALLOC_PER_NUMA_NODE=y
#
# Compile librte_cmdline

1
config/defconfig_x86_64-default-linuxapp-icc
View File

@ -186,7 +186,6 @@ CONFIG_RTE_LIBRTE_TIMER_DEBUG=n
CONFIG_RTE_LIBRTE_MALLOC=y
CONFIG_RTE_LIBRTE_MALLOC_DEBUG=n
CONFIG_RTE_MALLOC_MEMZONE_SIZE=11M
CONFIG_RTE_MALLOC_PER_NUMA_NODE=y
#
# Compile librte_cmdline

2
lib/librte_eal/common/Makefile
View File

@ -38,7 +38,7 @@ INC += rte_log.h rte_memcpy.h rte_memory.h rte_memzone.h rte_pci.h
INC += rte_pci_dev_ids.h rte_per_lcore.h rte_prefetch.h rte_random.h
INC += rte_rwlock.h rte_spinlock.h rte_tailq.h rte_interrupts.h rte_alarm.h
INC += rte_string_fns.h rte_cpuflags.h rte_version.h rte_tailq_elem.h
INC += rte_eal_memconfig.h
INC += rte_eal_memconfig.h rte_malloc_heap.h
ifeq ($(CONFIG_RTE_INSECURE_FUNCTION_WARNING),y)
INC += rte_warnings.h

56
lib/librte_eal/common/include/rte_malloc_heap.h Normal file
View File

@ -0,0 +1,56 @@
/*-
* BSD LICENSE
*
* Copyright(c) 2010-2012 Intel Corporation. All rights reserved.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#ifndef _RTE_MALLOC_HEAP_H_
#define _RTE_MALLOC_HEAP_H_
#include <stddef.h>
enum heap_state {
NOT_INITIALISED = 0,
INITIALISED
};
/**
* Structure to hold malloc heap
*/
struct malloc_heap {
enum heap_state volatile initialised;
unsigned numa_socket;
rte_spinlock_t lock;
struct malloc_elem * volatile free_head;
unsigned mz_count;
} __rte_cache_aligned;
#endif /* _RTE_MALLOC_HEAP_H_ */

31
lib/librte_malloc/malloc_heap.h
View File

@ -35,33 +35,28 @@
#ifndef MALLOC_HEAP_H_
#define MALLOC_HEAP_H_
enum heap_state {
NOT_INITIALISED = 0,
INITIALISED
};
#include <rte_malloc.h>
#include <rte_malloc_heap.h>
struct malloc_heap {
enum heap_state initialised;
unsigned numa_socket;
volatile unsigned mz_count;
rte_spinlock_t lock;
struct malloc_elem * volatile free_head;
} __rte_cache_aligned;
#define RTE_MALLOC_SOCKET_DEFAULT 0
#ifdef __cplusplus
extern "C" {
#endif
static inline unsigned
malloc_get_numa_socket(void)
{
unsigned malloc_socket = RTE_MALLOC_SOCKET_DEFAULT;
#ifdef RTE_MALLOC_PER_NUMA_NODE
malloc_socket = rte_socket_id();
#endif
return malloc_socket;
return rte_socket_id();
}
void *
malloc_heap_alloc(struct malloc_heap *heap, const char *type,
size_t size, unsigned align);
int
rte_eal_heap_memzone_init(void);
#ifdef __cplusplus
}
#endif
#endif /* MALLOC_HEAP_H_ */

54
lib/librte_malloc/rte_malloc.c
View File

@ -67,18 +67,47 @@ void rte_free(void *addr)
rte_panic("Fatal error: Invalid memory\n");
}
/*
* Allocate memory on specified heap.
*/
void *
rte_malloc_socket(const char *type, size_t size, unsigned align, int socket)
{
/* return NULL if size is 0 or alignment is not power-of-2 */
if (size == 0 || !rte_is_power_of_2(align))
return NULL;
if (socket == SOCKET_ID_ANY)
socket = malloc_get_numa_socket();
/* Check socket parameter */
if (socket >= RTE_MAX_NUMA_NODES)
return NULL;
return malloc_heap_alloc(&malloc_heaps[socket], type,
size, align == 0 ? 1 : align);
}
/*
* Allocate memory on default heap.
*/
void *
rte_malloc(const char *type, size_t size, unsigned align)
{
unsigned malloc_socket = malloc_get_numa_socket();
/* return NULL if size is 0 or alignment is not power-of-2 */
if (size == 0 || !rte_is_power_of_2(align))
return NULL;
return malloc_heap_alloc(&malloc_heap[malloc_socket], type,
size, align == 0 ? 1 : align);
return rte_malloc_socket(type, size, align, SOCKET_ID_ANY);
}
/*
* Allocate zero'd memory on specified heap.
*/
void *
rte_zmalloc_socket(const char *type, size_t size, unsigned align, int socket)
{
void *ptr = rte_malloc_socket(type, size, align, socket);
if (ptr != NULL)
memset(ptr, 0, size);
return ptr;
}
/*
@ -87,11 +116,16 @@ rte_malloc(const char *type, size_t size, unsigned align)
void *
rte_zmalloc(const char *type, size_t size, unsigned align)
{
void *ptr = rte_malloc(type, size, align);
return rte_zmalloc_socket(type, size, align, SOCKET_ID_ANY);
}
if (ptr != NULL)
memset(ptr, 0, size);
return ptr;
/*
* Allocate zero'd memory on specified heap.
*/
void *
rte_calloc_socket(const char *type, size_t num, size_t size, unsigned align, int socket)
{
return rte_zmalloc_socket(type, num * size, align, socket);
}
/*

96
lib/librte_malloc/rte_malloc.h
View File

@ -49,7 +49,8 @@ extern "C" {
/**
* This function allocates memory from the huge-page area of memory. The memory
* is not cleared.
* is not cleared. In NUMA systems, the memory allocated resides on the same
* NUMA socket as the core that calls this function.
*
* @param type
* A string identifying the type of allocated objects (useful for debug
@ -74,7 +75,8 @@ rte_malloc(const char *type, size_t size, unsigned align);
* Allocate zero'ed memory from the heap.
*
* Equivalent to rte_malloc() except that the memory zone is
* initialised with zeros.
* initialised with zeros. In NUMA systems, the memory allocated resides on the
* same NUMA socket as the core that calls this function.
*
* @param type
* A string identifying the type of allocated objects (useful for debug
@ -97,7 +99,8 @@ rte_zmalloc(const char *type, size_t size, unsigned align);
/**
* Replacement function for calloc(), using huge-page memory. Memory area is
* initialised with zeros.
* initialised with zeros. In NUMA systems, the memory allocated resides on the
* same NUMA socket as the core that calls this function.
*
* @param type
* A string identifying the type of allocated objects (useful for debug
@ -122,7 +125,8 @@ rte_calloc(const char *type, size_t num, size_t size, unsigned align);
/**
* Replacement function for realloc(), using huge-page memory. Reserved area
* memory is resized, preserving contents.
* memory is resized, preserving contents. In NUMA systems, the new area
* resides on the same NUMA socket as the old area.
*
* @param ptr
* Pointer to already allocated memory
@ -142,6 +146,88 @@ rte_calloc(const char *type, size_t num, size_t size, unsigned align);
void *
rte_realloc(void *ptr, size_t size, unsigned align);
/**
* This function allocates memory from the huge-page area of memory. The memory
* is not cleared.
*
* @param type
* A string identifying the type of allocated objects (useful for debug
* purposes, such as identifying the cause of a memory leak). Can be NULL.
* @param size
* Size (in bytes) to be allocated.
* @param align
* If 0, the return is a pointer that is suitably aligned for any kind of
* variable (in the same manner as malloc()).
* Otherwise, the return is a pointer that is a multiple of *align*. In
* this case, it must be a power of two. (Minimum alignment is the
* cacheline size, i.e. 64-bytes)
* @param socket
* NUMA socket to allocate memory on. If SOCKET_ID_ANY is used, this function
* will behave the same as rte_malloc().
* @return
* - NULL on error. Not enough memory, or invalid arguments (size is 0,
* align is not a power of two).
* - Otherwise, the pointer to the allocated object.
*/
void *
rte_malloc_socket(const char *type, size_t size, unsigned align, int socket);
/**
* Allocate zero'ed memory from the heap.
*
* Equivalent to rte_malloc() except that the memory zone is
* initialised with zeros.
*
* @param type
* A string identifying the type of allocated objects (useful for debug
* purposes, such as identifying the cause of a memory leak). Can be NULL.
* @param size
* Size (in bytes) to be allocated.
* @param align
* If 0, the return is a pointer that is suitably aligned for any kind of
* variable (in the same manner as malloc()).
* Otherwise, the return is a pointer that is a multiple of *align*. In
* this case, it must obviously be a power of two. (Minimum alignment is the
* cacheline size, i.e. 64-bytes)
* @param socket
* NUMA socket to allocate memory on. If SOCKET_ID_ANY is used, this function
* will behave the same as rte_zmalloc().
* @return
* - NULL on error. Not enough memory, or invalid arguments (size is 0,
* align is not a power of two).
* - Otherwise, the pointer to the allocated object.
*/
void *
rte_zmalloc_socket(const char *type, size_t size, unsigned align, int socket);
/**
* Replacement function for calloc(), using huge-page memory. Memory area is
* initialised with zeros.
*
* @param type
* A string identifying the type of allocated objects (useful for debug
* purposes, such as identifying the cause of a memory leak). Can be NULL.
* @param num
* Number of elements to be allocated.
* @param size
* Size (in bytes) of a single element.
* @param align
* If 0, the return is a pointer that is suitably aligned for any kind of
* variable (in the same manner as malloc()).
* Otherwise, the return is a pointer that is a multiple of *align*. In
* this case, it must obviously be a power of two. (Minimum alignment is the
* cacheline size, i.e. 64-bytes)
* @param socket
* NUMA socket to allocate memory on. If SOCKET_ID_ANY is used, this function
* will behave the same as rte_calloc().
* @return
* - NULL on error. Not enough memory, or invalid arguments (size is 0,
* align is not a power of two).
* - Otherwise, the pointer to the allocated object.
*/
void *
rte_calloc_socket(const char *type, size_t num, size_t size, unsigned align, int socket);
/**
* Frees the memory space pointed to by the provided pointer.
*
@ -193,6 +279,8 @@ rte_malloc_dump_stats(const char *type);
/**
* Set the maximum amount of allocated memory for this type.
*
* This is not yet implemented
*
* @param type
* A string identifying the type of allocated objects.
* @param max