d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Import jemalloc 37b6f95dcd866f51c91488531a2efc3ed4c2b754 (dev branch,

Browse Source 
prior to 3.0.0 release).  This version is likely very close to what will be
3.0.0.
This commit is contained in:
jasone 2012-05-10 18:29:40 +00:00
parent c42a8fec96
commit 136e1e4ddc
30 changed files with 1728 additions and 980 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
contrib/jemalloc/ChangeLog
View File

@ -19,9 +19,10 @@ found in the git revision history:
New features: New features:
- Implement Valgrind support, redzones, and quarantine. - Implement Valgrind support, redzones, and quarantine.
- Add support for additional operating systems: - Add support for additional platforms:
+ FreeBSD + FreeBSD
+ Mac OS X Lion + Mac OS X Lion
+ MinGW
- Add support for additional architectures: - Add support for additional architectures:
+ MIPS + MIPS
+ SH4 + SH4
@ -64,18 +65,24 @@ found in the git revision history:
- Remove the --enable-sysv configure option. - Remove the --enable-sysv configure option.
Bug fixes: Bug fixes:
- Fix fork-related bugs that could cause deadlock in children between fork
and exec.
- Fix a statistics-related bug in the "thread.arena" mallctl that could cause - Fix a statistics-related bug in the "thread.arena" mallctl that could cause
invalid statistics and crashes. invalid statistics and crashes.
- Work around TLS dallocation via free() on Linux. This bug could cause - Work around TLS deallocation via free() on Linux. This bug could cause
write-after-free memory corruption. write-after-free memory corruption.
- Fix a potential deadlock that could occur during interval- and
growth-triggered heap profile dumps.
- Fix chunk_alloc_dss() to stop claiming memory is zeroed. This bug could - Fix chunk_alloc_dss() to stop claiming memory is zeroed. This bug could
cause memory corruption and crashes with --enable-dss specified. cause memory corruption and crashes with --enable-dss specified.
- Fix fork-related bugs that could cause deadlock in children between fork
and exec.
- Fix malloc_stats_print() to honor 'b' and 'l' in the opts parameter. - Fix malloc_stats_print() to honor 'b' and 'l' in the opts parameter.
- Fix realloc(p, 0) to act like free(p). - Fix realloc(p, 0) to act like free(p).
- Do not enforce minimum alignment in memalign(). - Do not enforce minimum alignment in memalign().
- Check for NULL pointer in malloc_usable_size(). - Check for NULL pointer in malloc_usable_size().
- Fix an off-by-one heap profile statistics bug that could be observed in
interval- and growth-triggered heap profiles.
- Fix the "epoch" mallctl to update cached stats even if the passed in epoch
is 0.
- Fix bin->runcur management to fix a layout policy bug. This bug did not - Fix bin->runcur management to fix a layout policy bug. This bug did not
affect correctness. affect correctness.
- Fix a bug in choose_arena_hard() that potentially caused more arenas to be - Fix a bug in choose_arena_hard() that potentially caused more arenas to be

1
contrib/jemalloc/FREEBSD-Xlist
View File

@ -18,6 +18,7 @@ include/jemalloc/internal/jemalloc_internal.h.in
include/jemalloc/internal/size_classes.sh include/jemalloc/internal/size_classes.sh
include/jemalloc/jemalloc.h.in include/jemalloc/jemalloc.h.in
include/jemalloc/jemalloc_defs.h.in include/jemalloc/jemalloc_defs.h.in
include/msvc_compat/
install-sh install-sh
src/zone.c src/zone.c
test/ test/

63
contrib/jemalloc/FREEBSD-diffs
View File

@ -1,5 +1,5 @@
diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in
index e8a5722..cec85b5 100644 index 93c16dc..b5c5595 100644
--- a/doc/jemalloc.xml.in --- a/doc/jemalloc.xml.in
+++ b/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in
@@ -51,12 +51,23 @@ @@ -51,12 +51,23 @@
@ -27,7 +27,7 @@ index e8a5722..cec85b5 100644
<refsect2> <refsect2>
<title>Standard API</title> <title>Standard API</title>
<funcprototype> <funcprototype>
@@ -2091,4 +2102,16 @@ malloc_conf = "lg_chunk:24";]]></programlisting></para> @@ -2101,4 +2112,16 @@ malloc_conf = "lg_chunk:24";]]></programlisting></para>
<para>The <function>posix_memalign<parameter/></function> function conforms <para>The <function>posix_memalign<parameter/></function> function conforms
to IEEE Std 1003.1-2001 (&ldquo;POSIX.1&rdquo;).</para> to IEEE Std 1003.1-2001 (&ldquo;POSIX.1&rdquo;).</para>
</refsect1> </refsect1>
@ -45,7 +45,7 @@ index e8a5722..cec85b5 100644
+ </refsect1> + </refsect1>
</refentry> </refentry>
diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in
index b61abe8..edbb437 100644 index 268cd14..cfb1fb9 100644
--- a/include/jemalloc/internal/jemalloc_internal.h.in --- a/include/jemalloc/internal/jemalloc_internal.h.in
+++ b/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in
@@ -1,5 +1,8 @@ @@ -1,5 +1,8 @@
@ -54,12 +54,12 @@ index b61abe8..edbb437 100644
+#include "libc_private.h" +#include "libc_private.h"
+#include "namespace.h" +#include "namespace.h"
+ +
#include <sys/mman.h>
#include <sys/param.h>
#include <sys/syscall.h>
@@ -35,6 +38,9 @@
#include <pthread.h>
#include <math.h> #include <math.h>
#ifdef _WIN32
# include <windows.h>
@@ -54,6 +57,9 @@ typedef intptr_t ssize_t;
#endif
#include <fcntl.h>
+#include "un-namespace.h" +#include "un-namespace.h"
+#include "libc_private.h" +#include "libc_private.h"
@ -68,10 +68,10 @@ index b61abe8..edbb437 100644
#include "../jemalloc@install_suffix@.h" #include "../jemalloc@install_suffix@.h"
diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h
index 8837ef5..d7133f4 100644 index de44e14..564d604 100644
--- a/include/jemalloc/internal/mutex.h --- a/include/jemalloc/internal/mutex.h
+++ b/include/jemalloc/internal/mutex.h +++ b/include/jemalloc/internal/mutex.h
@@ -39,9 +39,6 @@ struct malloc_mutex_s { @@ -43,9 +43,6 @@ struct malloc_mutex_s {
#ifdef JEMALLOC_LAZY_LOCK #ifdef JEMALLOC_LAZY_LOCK
extern bool isthreaded; extern bool isthreaded;
@ -82,10 +82,10 @@ index 8837ef5..d7133f4 100644
bool malloc_mutex_init(malloc_mutex_t *mutex); bool malloc_mutex_init(malloc_mutex_t *mutex);
diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h diff --git a/include/jemalloc/internal/private_namespace.h b/include/jemalloc/internal/private_namespace.h
index bb1b63e..00eb169 100644 index b816647..b8ce6b1 100644
--- a/include/jemalloc/internal/private_namespace.h --- a/include/jemalloc/internal/private_namespace.h
+++ b/include/jemalloc/internal/private_namespace.h +++ b/include/jemalloc/internal/private_namespace.h
@@ -165,7 +165,6 @@ @@ -186,7 +186,6 @@
#define iqalloc JEMALLOC_N(iqalloc) #define iqalloc JEMALLOC_N(iqalloc)
#define iralloc JEMALLOC_N(iralloc) #define iralloc JEMALLOC_N(iralloc)
#define isalloc JEMALLOC_N(isalloc) #define isalloc JEMALLOC_N(isalloc)
@ -94,7 +94,7 @@ index bb1b63e..00eb169 100644
#define jemalloc_postfork_child JEMALLOC_N(jemalloc_postfork_child) #define jemalloc_postfork_child JEMALLOC_N(jemalloc_postfork_child)
#define jemalloc_postfork_parent JEMALLOC_N(jemalloc_postfork_parent) #define jemalloc_postfork_parent JEMALLOC_N(jemalloc_postfork_parent)
diff --git a/include/jemalloc/jemalloc.h.in b/include/jemalloc/jemalloc.h.in diff --git a/include/jemalloc/jemalloc.h.in b/include/jemalloc/jemalloc.h.in
index f0581db..f26d8bc 100644 index ad06948..505dd38 100644
--- a/include/jemalloc/jemalloc.h.in --- a/include/jemalloc/jemalloc.h.in
+++ b/include/jemalloc/jemalloc.h.in +++ b/include/jemalloc/jemalloc.h.in
@@ -15,6 +15,7 @@ extern "C" { @@ -15,6 +15,7 @@ extern "C" {
@ -107,10 +107,10 @@ index f0581db..f26d8bc 100644
#define ALLOCM_LG_ALIGN(la) (la) #define ALLOCM_LG_ALIGN(la) (la)
diff --git a/include/jemalloc/jemalloc_FreeBSD.h b/include/jemalloc/jemalloc_FreeBSD.h diff --git a/include/jemalloc/jemalloc_FreeBSD.h b/include/jemalloc/jemalloc_FreeBSD.h
new file mode 100644 new file mode 100644
index 0000000..2c5797f index 0000000..9efab93
--- /dev/null --- /dev/null
+++ b/include/jemalloc/jemalloc_FreeBSD.h +++ b/include/jemalloc/jemalloc_FreeBSD.h
@@ -0,0 +1,76 @@ @@ -0,0 +1,80 @@
+/* +/*
+ * Override settings that were generated in jemalloc_defs.h as necessary. + * Override settings that were generated in jemalloc_defs.h as necessary.
+ */ + */
@ -154,8 +154,12 @@ index 0000000..2c5797f
+# define LG_SIZEOF_PTR 2 +# define LG_SIZEOF_PTR 2
+#endif +#endif
+#ifdef __mips__ +#ifdef __mips__
+#ifdef __mips_n64
+# define LG_SIZEOF_PTR 3
+#else
+# define LG_SIZEOF_PTR 2 +# define LG_SIZEOF_PTR 2
+#endif +#endif
+#endif
+#ifdef __powerpc64__ +#ifdef __powerpc64__
+# define LG_SIZEOF_PTR 3 +# define LG_SIZEOF_PTR 3
+#elif defined(__powerpc__) +#elif defined(__powerpc__)
@ -188,20 +192,21 @@ index 0000000..2c5797f
+#define pthread_mutex_lock _pthread_mutex_lock +#define pthread_mutex_lock _pthread_mutex_lock
+#define pthread_mutex_unlock _pthread_mutex_unlock +#define pthread_mutex_unlock _pthread_mutex_unlock
diff --git a/src/jemalloc.c b/src/jemalloc.c diff --git a/src/jemalloc.c b/src/jemalloc.c
index f9c8916..8e24a5a 100644 index d42e91d..cdf6222 100644
--- a/src/jemalloc.c --- a/src/jemalloc.c
+++ b/src/jemalloc.c +++ b/src/jemalloc.c
@@ -8,6 +8,9 @@ malloc_tsd_data(, arenas, arena_t *, NULL) @@ -8,6 +8,10 @@ malloc_tsd_data(, arenas, arena_t *, NULL)
malloc_tsd_data(, thread_allocated, thread_allocated_t, malloc_tsd_data(, thread_allocated, thread_allocated_t,
THREAD_ALLOCATED_INITIALIZER) THREAD_ALLOCATED_INITIALIZER)
+const char *__malloc_options_1_0; +/* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */
+const char *__malloc_options_1_0 = NULL;
+__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); +__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
+ +
/* Runtime configuration options. */ /* Runtime configuration options. */
const char *je_malloc_conf JEMALLOC_ATTR(visibility("default")); const char *je_malloc_conf;
#ifdef JEMALLOC_DEBUG #ifdef JEMALLOC_DEBUG
@@ -401,7 +404,8 @@ malloc_conf_init(void) @@ -429,7 +433,8 @@ malloc_conf_init(void)
#endif #endif
; ;
@ -212,10 +217,10 @@ index f9c8916..8e24a5a 100644
* Do nothing; opts is already initialized to * Do nothing; opts is already initialized to
* the value of the MALLOC_CONF environment * the value of the MALLOC_CONF environment
diff --git a/src/mutex.c b/src/mutex.c diff --git a/src/mutex.c b/src/mutex.c
index 4b8ce57..7be5fc9 100644 index 37a843e..4a90a05 100644
--- a/src/mutex.c --- a/src/mutex.c
+++ b/src/mutex.c +++ b/src/mutex.c
@@ -63,6 +63,17 @@ pthread_create(pthread_t *__restrict thread, @@ -66,6 +66,17 @@ pthread_create(pthread_t *__restrict thread,
#ifdef JEMALLOC_MUTEX_INIT_CB #ifdef JEMALLOC_MUTEX_INIT_CB
int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
void *(calloc_cb)(size_t, size_t)); void *(calloc_cb)(size_t, size_t));
@ -234,14 +239,14 @@ index 4b8ce57..7be5fc9 100644
bool bool
diff --git a/src/util.c b/src/util.c diff --git a/src/util.c b/src/util.c
index 99ae26d..b80676c 100644 index 9b73c3e..f94799f 100644
--- a/src/util.c --- a/src/util.c
+++ b/src/util.c +++ b/src/util.c
@@ -60,6 +60,22 @@ wrtmessage(void *cbopaque, const char *s) @@ -58,6 +58,22 @@ wrtmessage(void *cbopaque, const char *s)
void (*je_malloc_message)(void *, const char *s)
JEMALLOC_ATTR(visibility("default")) = wrtmessage;
+JEMALLOC_CATTR(visibility("hidden"), static) JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s);
+JEMALLOC_ATTR(visibility("hidden"))
+void +void
+wrtmessage_1_0(const char *s1, const char *s2, const char *s3, +wrtmessage_1_0(const char *s1, const char *s2, const char *s3,
+ const char *s4) + const char *s4)
@ -258,5 +263,5 @@ index 99ae26d..b80676c 100644
+__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0); +__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0);
+ +
/* /*
* glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so * Wrapper around malloc_message() that avoids the need for
* provide a wrapper. * je_malloc_message(...) throughout the code.

2
contrib/jemalloc/VERSION
View File

@ -1 +1 @@
1.0.0-286-ga8f8d7540d66ddee7337db80c92890916e1063ca 1.0.0-335-g37b6f95dcd866f51c91488531a2efc3ed4c2b754

16
contrib/jemalloc/doc/jemalloc.3
View File

@ -2,12 +2,12 @@
.\" Title: JEMALLOC .\" Title: JEMALLOC
.\" Author: Jason Evans .\" Author: Jason Evans
.\" Generator: DocBook XSL Stylesheets v1.76.1 <http://docbook.sf.net/> .\" Generator: DocBook XSL Stylesheets v1.76.1 <http://docbook.sf.net/>
.\" Date: 04/21/2012 .\" Date: 05/09/2012
.\" Manual: User Manual .\" Manual: User Manual
.\" Source: jemalloc 1.0.0-286-ga8f8d7540d66ddee7337db80c92890916e1063ca .\" Source: jemalloc 1.0.0-335-g37b6f95dcd866f51c91488531a2efc3ed4c2b754
.\" Language: English .\" Language: English
.\" .\"
.TH "JEMALLOC" "3" "04/21/2012" "jemalloc 1.0.0-286-ga8f8d7540d" "User Manual" .TH "JEMALLOC" "3" "05/09/2012" "jemalloc 1.0.0-335-g37b6f95dcd" "User Manual"
.\" ----------------------------------------------------------------- .\" -----------------------------------------------------------------
.\" * Define some portability stuff .\" * Define some portability stuff
.\" ----------------------------------------------------------------- .\" -----------------------------------------------------------------
@ -31,7 +31,7 @@
jemalloc \- general purpose memory allocation functions jemalloc \- general purpose memory allocation functions
.SH "LIBRARY" .SH "LIBRARY"
.PP .PP
This manual describes jemalloc 1\&.0\&.0\-286\-ga8f8d7540d66ddee7337db80c92890916e1063ca\&. More information can be found at the This manual describes jemalloc 1\&.0\&.0\-335\-g37b6f95dcd866f51c91488531a2efc3ed4c2b754\&. More information can be found at the
\m[blue]\fBjemalloc website\fR\m[]\&\s-2\u[1]\d\s+2\&. \m[blue]\fBjemalloc website\fR\m[]\&\s-2\u[1]\d\s+2\&.
.PP .PP
The following configuration options are enabled in libc\*(Aqs built\-in jemalloc: The following configuration options are enabled in libc\*(Aqs built\-in jemalloc:
@ -567,6 +567,12 @@ was specified during build configuration\&.
was specified during build configuration\&. was specified during build configuration\&.
.RE .RE
.PP .PP
"config\&.mremap" (\fBbool\fR) r\-
.RS 4
\fB\-\-enable\-mremap\fR
was specified during build configuration\&.
.RE
.PP
"config\&.munmap" (\fBbool\fR) r\- "config\&.munmap" (\fBbool\fR) r\-
.RS 4 .RS 4
\fB\-\-enable\-munmap\fR \fB\-\-enable\-munmap\fR
@ -1462,7 +1468,7 @@ jemalloc website
.IP " 2." 4 .IP " 2." 4
Valgrind Valgrind
.RS 4 .RS 4
\%http://http://valgrind.org/ \%http://valgrind.org/
.RE .RE
.IP " 3." 4 .IP " 3." 4
gperftools package gperftools package

401
contrib/jemalloc/include/jemalloc/internal/arena.h
View File

@ -109,7 +109,8 @@ struct arena_chunk_map_s {
* *
* p : run page offset * p : run page offset
* s : run size * s : run size
* c : (binind+1) for size class (used only if prof_promote is true) * n : binind for size class; large objects set these to BININD_INVALID
* except for promoted allocations (see prof_promote)
* x : don't care * x : don't care
* - : 0 * - : 0
* + : 1 * + : 1
@ -117,35 +118,38 @@ struct arena_chunk_map_s {
* [dula] : bit unset * [dula] : bit unset
* *
* Unallocated (clean): * Unallocated (clean):
* ssssssss ssssssss ssss---- ----du-a * ssssssss ssssssss ssss1111 1111du-a
* xxxxxxxx xxxxxxxx xxxx---- -----Uxx * xxxxxxxx xxxxxxxx xxxxxxxx xxxx-Uxx
* ssssssss ssssssss ssss---- ----dU-a * ssssssss ssssssss ssss1111 1111dU-a
* *
* Unallocated (dirty): * Unallocated (dirty):
* ssssssss ssssssss ssss---- ----D--a * ssssssss ssssssss ssss1111 1111D--a
* xxxxxxxx xxxxxxxx xxxx---- ----xxxx * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
* ssssssss ssssssss ssss---- ----D--a * ssssssss ssssssss ssss1111 1111D--a
* *
* Small: * Small:
* pppppppp pppppppp pppp---- ----d--A * pppppppp pppppppp ppppnnnn nnnnd--A
* pppppppp pppppppp pppp---- -------A * pppppppp pppppppp ppppnnnn nnnn---A
* pppppppp pppppppp pppp---- ----d--A * pppppppp pppppppp ppppnnnn nnnnd--A
* *
* Large: * Large:
* ssssssss ssssssss ssss---- ----D-LA * ssssssss ssssssss ssss1111 1111D-LA
* xxxxxxxx xxxxxxxx xxxx---- ----xxxx * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx
* -------- -------- -------- ----D-LA * -------- -------- ----1111 1111D-LA
* *
* Large (sampled, size <= PAGE): * Large (sampled, size <= PAGE):
* ssssssss ssssssss sssscccc ccccD-LA * ssssssss ssssssss ssssnnnn nnnnD-LA
* *
* Large (not sampled, size == PAGE): * Large (not sampled, size == PAGE):
* ssssssss ssssssss ssss---- ----D-LA * ssssssss ssssssss ssss1111 1111D-LA
*/ */
size_t bits; size_t bits;
#define CHUNK_MAP_CLASS_SHIFT 4 #define CHUNK_MAP_BININD_SHIFT 4
#define CHUNK_MAP_CLASS_MASK ((size_t)0xff0U) #define BININD_INVALID ((size_t)0xffU)
#define CHUNK_MAP_FLAGS_MASK ((size_t)0xfU) /* CHUNK_MAP_BININD_MASK == (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) */
#define CHUNK_MAP_BININD_MASK ((size_t)0xff0U)
#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK
#define CHUNK_MAP_FLAGS_MASK ((size_t)0xcU)
#define CHUNK_MAP_DIRTY ((size_t)0x8U) #define CHUNK_MAP_DIRTY ((size_t)0x8U)
#define CHUNK_MAP_UNZEROED ((size_t)0x4U) #define CHUNK_MAP_UNZEROED ((size_t)0x4U)
#define CHUNK_MAP_LARGE ((size_t)0x2U) #define CHUNK_MAP_LARGE ((size_t)0x2U)
@ -409,8 +413,14 @@ void *arena_malloc_small(arena_t *arena, size_t size, bool zero);
void *arena_malloc_large(arena_t *arena, size_t size, bool zero); void *arena_malloc_large(arena_t *arena, size_t size, bool zero);
void *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); void *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero);
void arena_prof_promoted(const void *ptr, size_t size); void arena_prof_promoted(const void *ptr, size_t size);
void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, void arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr,
arena_chunk_map_t *mapelm); arena_chunk_map_t *mapelm);
void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
size_t pageind, arena_chunk_map_t *mapelm);
void arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
size_t pageind);
void arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk,
void *ptr);
void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr);
void arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, void arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
arena_stats_t *astats, malloc_bin_stats_t *bstats, arena_stats_t *astats, malloc_bin_stats_t *bstats,
@ -430,6 +440,31 @@ void arena_postfork_child(arena_t *arena);
#ifdef JEMALLOC_H_INLINES #ifdef JEMALLOC_H_INLINES
#ifndef JEMALLOC_ENABLE_INLINE #ifndef JEMALLOC_ENABLE_INLINE
arena_chunk_map_t *arena_mapp_get(arena_chunk_t *chunk, size_t pageind);
size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk,
size_t pageind);
size_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind);
size_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind);
void arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind,
size_t size, size_t flags);
void arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
size_t size);
void arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind,
size_t size, size_t flags);
void arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind,
size_t binind);
void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind,
size_t runind, size_t binind, size_t flags);
void arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
size_t unzeroed);
size_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits);
size_t arena_bin_index(arena_t *arena, arena_bin_t *bin); size_t arena_bin_index(arena_t *arena, arena_bin_t *bin);
unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
const void *ptr); const void *ptr);
@ -442,6 +477,227 @@ void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr,
#endif #endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
# ifdef JEMALLOC_ARENA_INLINE_A
JEMALLOC_INLINE arena_chunk_map_t *
arena_mapp_get(arena_chunk_t *chunk, size_t pageind)
{
assert(pageind >= map_bias);
assert(pageind < chunk_npages);
return (&chunk->map[pageind-map_bias]);
}
JEMALLOC_INLINE size_t *
arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind)
{
return (&arena_mapp_get(chunk, pageind)->bits);
}
JEMALLOC_INLINE size_t
arena_mapbits_get(arena_chunk_t *chunk, size_t pageind)
{
return (*arena_mapbitsp_get(chunk, pageind));
}
JEMALLOC_INLINE size_t
arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0);
return (mapbits & ~PAGE_MASK);
}
JEMALLOC_INLINE size_t
arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) ==
(CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED));
return (mapbits & ~PAGE_MASK);
}
JEMALLOC_INLINE size_t
arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) ==
CHUNK_MAP_ALLOCATED);
return (mapbits >> LG_PAGE);
}
JEMALLOC_INLINE size_t
arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
size_t binind;
mapbits = arena_mapbits_get(chunk, pageind);
binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT;
assert(binind < NBINS || binind == BININD_INVALID);
return (binind);
}
JEMALLOC_INLINE size_t
arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
return (mapbits & CHUNK_MAP_DIRTY);
}
JEMALLOC_INLINE size_t
arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
return (mapbits & CHUNK_MAP_UNZEROED);
}
JEMALLOC_INLINE size_t
arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
return (mapbits & CHUNK_MAP_LARGE);
}
JEMALLOC_INLINE size_t
arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind)
{
size_t mapbits;
mapbits = arena_mapbits_get(chunk, pageind);
return (mapbits & CHUNK_MAP_ALLOCATED);
}
JEMALLOC_INLINE void
arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size,
size_t flags)
{
size_t *mapbitsp;
mapbitsp = arena_mapbitsp_get(chunk, pageind);
assert((size & PAGE_MASK) == 0);
assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0);
*mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags;
}
JEMALLOC_INLINE void
arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind,
size_t size)
{
size_t *mapbitsp;
mapbitsp = arena_mapbitsp_get(chunk, pageind);
assert((size & PAGE_MASK) == 0);
assert((*mapbitsp & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0);
*mapbitsp = size | (*mapbitsp & PAGE_MASK);
}
JEMALLOC_INLINE void
arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size,
size_t flags)
{
size_t *mapbitsp;
mapbitsp = arena_mapbitsp_get(chunk, pageind);
assert((size & PAGE_MASK) == 0);
assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0);
*mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags | CHUNK_MAP_LARGE |
CHUNK_MAP_ALLOCATED;
}
JEMALLOC_INLINE void
arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind,
size_t binind)
{
size_t *mapbitsp;
assert(binind <= BININD_INVALID);
mapbitsp = arena_mapbitsp_get(chunk, pageind);
assert(arena_mapbits_large_size_get(chunk, pageind) == PAGE);
*mapbitsp = (*mapbitsp & ~CHUNK_MAP_BININD_MASK) | (binind <<
CHUNK_MAP_BININD_SHIFT);
}
JEMALLOC_INLINE void
arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind,
size_t binind, size_t flags)
{
size_t *mapbitsp;
assert(binind < BININD_INVALID);
mapbitsp = arena_mapbitsp_get(chunk, pageind);
assert(pageind - runind >= map_bias);
assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0);
*mapbitsp = (runind << LG_PAGE) | (binind << CHUNK_MAP_BININD_SHIFT) |
flags | CHUNK_MAP_ALLOCATED;
}
JEMALLOC_INLINE void
arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind,
size_t unzeroed)
{
size_t *mapbitsp;
mapbitsp = arena_mapbitsp_get(chunk, pageind);
*mapbitsp = (*mapbitsp & ~CHUNK_MAP_UNZEROED) | unzeroed;
}
JEMALLOC_INLINE size_t
arena_ptr_small_binind_get(const void *ptr, size_t mapbits)
{
size_t binind;
binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT;
if (config_debug) {
arena_chunk_t *chunk;
arena_t *arena;
size_t pageind;
size_t actual_mapbits;
arena_run_t *run;
arena_bin_t *bin;
size_t actual_binind;
arena_bin_info_t *bin_info;
assert(binind != BININD_INVALID);
assert(binind < NBINS);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
arena = chunk->arena;
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
actual_mapbits = arena_mapbits_get(chunk, pageind);
assert(mapbits == actual_mapbits);
assert(arena_mapbits_large_get(chunk, pageind) == 0);
assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind -
(actual_mapbits >> LG_PAGE)) << LG_PAGE));
bin = run->bin;
actual_binind = bin - arena->bins;
assert(binind == actual_binind);
bin_info = &arena_bin_info[actual_binind];
assert(((uintptr_t)ptr - ((uintptr_t)run +
(uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval
== 0);
}
return (binind);
}
# endif /* JEMALLOC_ARENA_INLINE_A */
# ifdef JEMALLOC_ARENA_INLINE_B
JEMALLOC_INLINE size_t JEMALLOC_INLINE size_t
arena_bin_index(arena_t *arena, arena_bin_t *bin) arena_bin_index(arena_t *arena, arena_bin_t *bin)
{ {
@ -535,7 +791,7 @@ arena_prof_ctx_get(const void *ptr)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
mapbits = chunk->map[pageind-map_bias].bits; mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
if ((mapbits & CHUNK_MAP_LARGE) == 0) { if ((mapbits & CHUNK_MAP_LARGE) == 0) {
if (prof_promote) if (prof_promote)
@ -544,7 +800,8 @@ arena_prof_ctx_get(const void *ptr)
arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
(uintptr_t)((pageind - (mapbits >> LG_PAGE)) << (uintptr_t)((pageind - (mapbits >> LG_PAGE)) <<
LG_PAGE)); LG_PAGE));
size_t binind = arena_bin_index(chunk->arena, run->bin); size_t binind = arena_ptr_small_binind_get(ptr,
mapbits);
arena_bin_info_t *bin_info = &arena_bin_info[binind]; arena_bin_info_t *bin_info = &arena_bin_info[binind];
unsigned regind; unsigned regind;
@ -554,7 +811,7 @@ arena_prof_ctx_get(const void *ptr)
sizeof(prof_ctx_t *))); sizeof(prof_ctx_t *)));
} }
} else } else
ret = chunk->map[pageind-map_bias].prof_ctx; ret = arena_mapp_get(chunk, pageind)->prof_ctx;
return (ret); return (ret);
} }
@ -571,19 +828,18 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
mapbits = chunk->map[pageind-map_bias].bits; mapbits = arena_mapbits_get(chunk, pageind);
assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
if ((mapbits & CHUNK_MAP_LARGE) == 0) { if ((mapbits & CHUNK_MAP_LARGE) == 0) {
if (prof_promote == false) { if (prof_promote == false) {
arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
(uintptr_t)((pageind - (mapbits >> LG_PAGE)) << (uintptr_t)((pageind - (mapbits >> LG_PAGE)) <<
LG_PAGE)); LG_PAGE));
arena_bin_t *bin = run->bin;
size_t binind; size_t binind;
arena_bin_info_t *bin_info; arena_bin_info_t *bin_info;
unsigned regind; unsigned regind;
binind = arena_bin_index(chunk->arena, bin); binind = arena_ptr_small_binind_get(ptr, mapbits);
bin_info = &arena_bin_info[binind]; bin_info = &arena_bin_info[binind];
regind = arena_run_regind(run, bin_info, ptr); regind = arena_run_regind(run, bin_info, ptr);
@ -592,7 +848,7 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
} else } else
assert((uintptr_t)ctx == (uintptr_t)1U); assert((uintptr_t)ctx == (uintptr_t)1U);
} else } else
chunk->map[pageind-map_bias].prof_ctx = ctx; arena_mapp_get(chunk, pageind)->prof_ctx = ctx;
} }
JEMALLOC_INLINE void * JEMALLOC_INLINE void *
@ -631,35 +887,42 @@ arena_salloc(const void *ptr, bool demote)
{ {
size_t ret; size_t ret;
arena_chunk_t *chunk; arena_chunk_t *chunk;
size_t pageind, mapbits; size_t pageind, binind;
assert(ptr != NULL); assert(ptr != NULL);
assert(CHUNK_ADDR2BASE(ptr) != ptr); assert(CHUNK_ADDR2BASE(ptr) != ptr);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
mapbits = chunk->map[pageind-map_bias].bits; assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); binind = arena_mapbits_binind_get(chunk, pageind);
if ((mapbits & CHUNK_MAP_LARGE) == 0) { if (binind == BININD_INVALID || (config_prof && demote == false &&
arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + prof_promote && arena_mapbits_large_get(chunk, pageind) != 0)) {
(uintptr_t)((pageind - (mapbits >> LG_PAGE)) << LG_PAGE)); /*
size_t binind = arena_bin_index(chunk->arena, run->bin); * Large allocation. In the common case (demote == true), and
arena_bin_info_t *bin_info = &arena_bin_info[binind]; * as this is an inline function, most callers will only end up
assert(((uintptr_t)ptr - ((uintptr_t)run + * looking at binind to determine that ptr is a small
(uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval * allocation.
== 0); */
ret = bin_info->reg_size;
} else {
assert(((uintptr_t)ptr & PAGE_MASK) == 0); assert(((uintptr_t)ptr & PAGE_MASK) == 0);
ret = mapbits & ~PAGE_MASK; ret = arena_mapbits_large_size_get(chunk, pageind);
if (config_prof && demote && prof_promote && ret == PAGE &&
(mapbits & CHUNK_MAP_CLASS_MASK) != 0) {
size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >>
CHUNK_MAP_CLASS_SHIFT) - 1;
assert(binind < NBINS);
ret = arena_bin_info[binind].reg_size;
}
assert(ret != 0); assert(ret != 0);
assert(pageind + (ret>>LG_PAGE) <= chunk_npages);
assert(ret == PAGE || arena_mapbits_large_size_get(chunk,
pageind+(ret>>LG_PAGE)-1) == 0);
assert(binind == arena_mapbits_binind_get(chunk,
pageind+(ret>>LG_PAGE)-1));
assert(arena_mapbits_dirty_get(chunk, pageind) ==
arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1));
} else {
/*
* Small allocation (possibly promoted to a large object due to
* prof_promote).
*/
assert(arena_mapbits_large_get(chunk, pageind) != 0 ||
arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk,
pageind)) == binind);
ret = arena_bin_info[binind].reg_size;
} }
return (ret); return (ret);
@ -668,8 +931,7 @@ arena_salloc(const void *ptr, bool demote)
JEMALLOC_INLINE void JEMALLOC_INLINE void
arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache) arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache)
{ {
size_t pageind; size_t pageind, mapbits;
arena_chunk_map_t *mapelm;
tcache_t *tcache; tcache_t *tcache;
assert(arena != NULL); assert(arena != NULL);
@ -678,47 +940,30 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache)
assert(CHUNK_ADDR2BASE(ptr) != ptr); assert(CHUNK_ADDR2BASE(ptr) != ptr);
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
mapelm = &chunk->map[pageind-map_bias]; mapbits = arena_mapbits_get(chunk, pageind);
assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0); assert(arena_mapbits_allocated_get(chunk, pageind) != 0);
if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) { if ((mapbits & CHUNK_MAP_LARGE) == 0) {
/* Small allocation. */ /* Small allocation. */
if (try_tcache && (tcache = tcache_get(false)) != NULL) if (try_tcache && (tcache = tcache_get(false)) != NULL) {
tcache_dalloc_small(tcache, ptr); size_t binind;
else {
arena_run_t *run;
arena_bin_t *bin;
run = (arena_run_t *)((uintptr_t)chunk + binind = arena_ptr_small_binind_get(ptr, mapbits);
(uintptr_t)((pageind - (mapelm->bits >> LG_PAGE)) << tcache_dalloc_small(tcache, ptr, binind);
LG_PAGE)); } else
bin = run->bin; arena_dalloc_small(arena, chunk, ptr, pageind);
if (config_debug) {
size_t binind = arena_bin_index(arena, bin);
UNUSED arena_bin_info_t *bin_info =
&arena_bin_info[binind];
assert(((uintptr_t)ptr - ((uintptr_t)run +
(uintptr_t)bin_info->reg0_offset)) %
bin_info->reg_interval == 0);
}
malloc_mutex_lock(&bin->lock);
arena_dalloc_bin(arena, chunk, ptr, mapelm);
malloc_mutex_unlock(&bin->lock);
}
} else { } else {
size_t size = mapelm->bits & ~PAGE_MASK; size_t size = arena_mapbits_large_size_get(chunk, pageind);
assert(((uintptr_t)ptr & PAGE_MASK) == 0); assert(((uintptr_t)ptr & PAGE_MASK) == 0);
if (try_tcache && size <= tcache_maxclass && (tcache = if (try_tcache && size <= tcache_maxclass && (tcache =
tcache_get(false)) != NULL) { tcache_get(false)) != NULL) {
tcache_dalloc_large(tcache, ptr, size); tcache_dalloc_large(tcache, ptr, size);
} else { } else
malloc_mutex_lock(&arena->lock);
arena_dalloc_large(arena, chunk, ptr); arena_dalloc_large(arena, chunk, ptr);
malloc_mutex_unlock(&arena->lock);
}
} }
} }
# endif /* JEMALLOC_ARENA_INLINE_B */
#endif #endif
#endif /* JEMALLOC_H_INLINES */ #endif /* JEMALLOC_H_INLINES */

28
contrib/jemalloc/include/jemalloc/internal/atomic.h
View File

@ -47,6 +47,20 @@ atomic_sub_uint64(uint64_t *p, uint64_t x)
return (__sync_sub_and_fetch(p, x)); return (__sync_sub_and_fetch(p, x));
} }
#elif (defined(_MSC_VER))
JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x)
{
return (InterlockedExchangeAdd64(p, x));
}
JEMALLOC_INLINE uint64_t
atomic_sub_uint64(uint64_t *p, uint64_t x)
{
return (InterlockedExchangeAdd64(p, -((int64_t)x)));
}
#elif (defined(JEMALLOC_OSATOMIC)) #elif (defined(JEMALLOC_OSATOMIC))
JEMALLOC_INLINE uint64_t JEMALLOC_INLINE uint64_t
atomic_add_uint64(uint64_t *p, uint64_t x) atomic_add_uint64(uint64_t *p, uint64_t x)
@ -145,6 +159,20 @@ atomic_sub_uint32(uint32_t *p, uint32_t x)
return (__sync_sub_and_fetch(p, x)); return (__sync_sub_and_fetch(p, x));
} }
#elif (defined(_MSC_VER))
JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x)
{
return (InterlockedExchangeAdd(p, x));
}
JEMALLOC_INLINE uint32_t
atomic_sub_uint32(uint32_t *p, uint32_t x)
{
return (InterlockedExchangeAdd(p, -((int32_t)x)));
}
#elif (defined(JEMALLOC_OSATOMIC)) #elif (defined(JEMALLOC_OSATOMIC))
JEMALLOC_INLINE uint32_t JEMALLOC_INLINE uint32_t
atomic_add_uint32(uint32_t *p, uint32_t x) atomic_add_uint32(uint32_t *p, uint32_t x)

31
contrib/jemalloc/include/jemalloc/internal/ctl.h
View File

@ -2,6 +2,8 @@
#ifdef JEMALLOC_H_TYPES #ifdef JEMALLOC_H_TYPES
typedef struct ctl_node_s ctl_node_t; typedef struct ctl_node_s ctl_node_t;
typedef struct ctl_named_node_s ctl_named_node_t;
typedef struct ctl_indexed_node_s ctl_indexed_node_t;
typedef struct ctl_arena_stats_s ctl_arena_stats_t; typedef struct ctl_arena_stats_s ctl_arena_stats_t;
typedef struct ctl_stats_s ctl_stats_t; typedef struct ctl_stats_s ctl_stats_t;
@ -11,20 +13,21 @@ typedef struct ctl_stats_s ctl_stats_t;
struct ctl_node_s { struct ctl_node_s {
bool named; bool named;
union { };
struct {
const char *name; struct ctl_named_node_s {
/* If (nchildren == 0), this is a terminal node. */ struct ctl_node_s node;
unsigned nchildren; const char *name;
const ctl_node_t *children; /* If (nchildren == 0), this is a terminal node. */
} named; unsigned nchildren;
struct { const ctl_node_t *children;
const ctl_node_t *(*index)(const size_t *, size_t, int (*ctl)(const size_t *, size_t, void *, size_t *,
size_t); void *, size_t);
} indexed; };
} u;
int (*ctl)(const size_t *, size_t, void *, size_t *, void *, struct ctl_indexed_node_s {
size_t); struct ctl_node_s node;
const ctl_named_node_t *(*index)(const size_t *, size_t, size_t);
}; };
struct ctl_arena_stats_s { struct ctl_arena_stats_s {

89
contrib/jemalloc/include/jemalloc/internal/jemalloc_internal.h
View File

@ -3,23 +3,34 @@
#include "libc_private.h" #include "libc_private.h"
#include "namespace.h" #include "namespace.h"
#include <sys/mman.h> #include <math.h>
#include <sys/param.h> #ifdef _WIN32
#include <sys/syscall.h> # include <windows.h>
#if !defined(SYS_write) && defined(__NR_write) # define ENOENT ERROR_PATH_NOT_FOUND
#define SYS_write __NR_write # define EINVAL ERROR_BAD_ARGUMENTS
# define EAGAIN ERROR_OUTOFMEMORY
# define EPERM ERROR_WRITE_FAULT
# define EFAULT ERROR_INVALID_ADDRESS
# define ENOMEM ERROR_NOT_ENOUGH_MEMORY
# undef ERANGE
# define ERANGE ERROR_INVALID_DATA
#else
# include <sys/param.h>
# include <sys/mman.h>
# include <sys/syscall.h>
# if !defined(SYS_write) && defined(__NR_write)
# define SYS_write __NR_write
# endif
# include <sys/uio.h>
# include <pthread.h>
# include <errno.h>
#endif #endif
#include <sys/time.h>
#include <sys/types.h> #include <sys/types.h>
#include <sys/uio.h>
#include <errno.h>
#include <limits.h> #include <limits.h>
#ifndef SIZE_T_MAX #ifndef SIZE_T_MAX
# define SIZE_T_MAX SIZE_MAX # define SIZE_T_MAX SIZE_MAX
#endif #endif
#include <pthread.h>
#include <sched.h>
#include <stdarg.h> #include <stdarg.h>
#include <stdbool.h> #include <stdbool.h>
#include <stdio.h> #include <stdio.h>
@ -33,10 +44,18 @@
#include <string.h> #include <string.h>
#include <strings.h> #include <strings.h>
#include <ctype.h> #include <ctype.h>
#include <unistd.h> #ifdef _MSC_VER
# include <io.h>
typedef intptr_t ssize_t;
# define PATH_MAX 1024
# define STDERR_FILENO 2
# define __func__ __FUNCTION__
/* Disable warnings about deprecated system functions */
# pragma warning(disable: 4996)
#else
# include <unistd.h>
#endif
#include <fcntl.h> #include <fcntl.h>
#include <pthread.h>
#include <math.h>
#include "un-namespace.h" #include "un-namespace.h"
#include "libc_private.h" #include "libc_private.h"
@ -110,6 +129,13 @@ static const bool config_prof_libunwind =
false false
#endif #endif
; ;
static const bool config_mremap =
#ifdef JEMALLOC_MREMAP
true
#else
false
#endif
;
static const bool config_munmap = static const bool config_munmap =
#ifdef JEMALLOC_MUNMAP #ifdef JEMALLOC_MUNMAP
true true
@ -218,6 +244,9 @@ static const bool config_ivsalloc =
#else #else
# define JEMALLOC_ENABLE_INLINE # define JEMALLOC_ENABLE_INLINE
# define JEMALLOC_INLINE static inline # define JEMALLOC_INLINE static inline
# ifdef _MSC_VER
# define inline _inline
# endif
#endif #endif
/* Smallest size class to support. */ /* Smallest size class to support. */
@ -229,7 +258,7 @@ static const bool config_ivsalloc =
* classes). * classes).
*/ */
#ifndef LG_QUANTUM #ifndef LG_QUANTUM
# ifdef __i386__ # if (defined(__i386__) || defined(_M_IX86))
# define LG_QUANTUM 4 # define LG_QUANTUM 4
# endif # endif
# ifdef __ia64__ # ifdef __ia64__
@ -241,7 +270,7 @@ static const bool config_ivsalloc =
# ifdef __sparc64__ # ifdef __sparc64__
# define LG_QUANTUM 4 # define LG_QUANTUM 4
# endif # endif
# if (defined(__amd64__) || defined(__x86_64__)) # if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64))
# define LG_QUANTUM 4 # define LG_QUANTUM 4
# endif # endif
# ifdef __arm__ # ifdef __arm__
@ -291,9 +320,12 @@ static const bool config_ivsalloc =
/* /*
* Maximum size of L1 cache line. This is used to avoid cache line aliasing. * Maximum size of L1 cache line. This is used to avoid cache line aliasing.
* In addition, this controls the spacing of cacheline-spaced size classes. * In addition, this controls the spacing of cacheline-spaced size classes.
*
* CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can
* only handle raw constants.
*/ */
#define LG_CACHELINE 6 #define LG_CACHELINE 6
#define CACHELINE ((size_t)(1U << LG_CACHELINE)) #define CACHELINE 64
#define CACHELINE_MASK (CACHELINE - 1) #define CACHELINE_MASK (CACHELINE - 1)
/* Return the smallest cacheline multiple that is >= s. */ /* Return the smallest cacheline multiple that is >= s. */
@ -324,6 +356,20 @@ static const bool config_ivsalloc =
#define ALIGNMENT_CEILING(s, alignment) \ #define ALIGNMENT_CEILING(s, alignment) \
(((s) + (alignment - 1)) & (-(alignment))) (((s) + (alignment - 1)) & (-(alignment)))
/* Declare a variable length array */
#if __STDC_VERSION__ < 199901L
# ifdef _MSC_VER
# include <malloc.h>
# define alloca _alloca
# else
# include <alloca.h>
# endif
# define VARIABLE_ARRAY(type, name, count) \
type *name = alloca(sizeof(type) * count)
#else
# define VARIABLE_ARRAY(type, name, count) type name[count]
#endif
#ifdef JEMALLOC_VALGRIND #ifdef JEMALLOC_VALGRIND
/* /*
* The JEMALLOC_VALGRIND_*() macros must be macros rather than functions * The JEMALLOC_VALGRIND_*() macros must be macros rather than functions
@ -655,8 +701,17 @@ choose_arena(arena_t *arena)
#include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/bitmap.h"
#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/rtree.h"
#include "jemalloc/internal/tcache.h" /*
* Include arena.h twice in order to resolve circular dependencies with
* tcache.h.
*/
#define JEMALLOC_ARENA_INLINE_A
#include "jemalloc/internal/arena.h" #include "jemalloc/internal/arena.h"
#undef JEMALLOC_ARENA_INLINE_A
#include "jemalloc/internal/tcache.h"
#define JEMALLOC_ARENA_INLINE_B
#include "jemalloc/internal/arena.h"
#undef JEMALLOC_ARENA_INLINE_B
#include "jemalloc/internal/hash.h" #include "jemalloc/internal/hash.h"
#include "jemalloc/internal/quarantine.h" #include "jemalloc/internal/quarantine.h"

20
contrib/jemalloc/include/jemalloc/internal/mutex.h
View File

@ -3,10 +3,12 @@
typedef struct malloc_mutex_s malloc_mutex_t; typedef struct malloc_mutex_s malloc_mutex_t;
#ifdef JEMALLOC_OSSPIN #ifdef _WIN32
#define MALLOC_MUTEX_INITIALIZER {0} # define MALLOC_MUTEX_INITIALIZER
#elif (defined(JEMALLOC_OSSPIN))
# define MALLOC_MUTEX_INITIALIZER {0}
#elif (defined(JEMALLOC_MUTEX_INIT_CB)) #elif (defined(JEMALLOC_MUTEX_INIT_CB))
#define MALLOC_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER, NULL} # define MALLOC_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER, NULL}
#else #else
# if (defined(PTHREAD_MUTEX_ADAPTIVE_NP) && \ # if (defined(PTHREAD_MUTEX_ADAPTIVE_NP) && \
defined(PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP)) defined(PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP))
@ -23,7 +25,9 @@ typedef struct malloc_mutex_s malloc_mutex_t;
#ifdef JEMALLOC_H_STRUCTS #ifdef JEMALLOC_H_STRUCTS
struct malloc_mutex_s { struct malloc_mutex_s {
#ifdef JEMALLOC_OSSPIN #ifdef _WIN32
CRITICAL_SECTION lock;
#elif (defined(JEMALLOC_OSSPIN))
OSSpinLock lock; OSSpinLock lock;
#elif (defined(JEMALLOC_MUTEX_INIT_CB)) #elif (defined(JEMALLOC_MUTEX_INIT_CB))
pthread_mutex_t lock; pthread_mutex_t lock;
@ -62,7 +66,9 @@ malloc_mutex_lock(malloc_mutex_t *mutex)
{ {
if (isthreaded) { if (isthreaded) {
#ifdef JEMALLOC_OSSPIN #ifdef _WIN32
EnterCriticalSection(&mutex->lock);
#elif (defined(JEMALLOC_OSSPIN))
OSSpinLockLock(&mutex->lock); OSSpinLockLock(&mutex->lock);
#else #else
pthread_mutex_lock(&mutex->lock); pthread_mutex_lock(&mutex->lock);
@ -75,7 +81,9 @@ malloc_mutex_unlock(malloc_mutex_t *mutex)
{ {
if (isthreaded) { if (isthreaded) {
#ifdef JEMALLOC_OSSPIN #ifdef _WIN32
LeaveCriticalSection(&mutex->lock);
#elif (defined(JEMALLOC_OSSPIN))
OSSpinLockUnlock(&mutex->lock); OSSpinLockUnlock(&mutex->lock);
#else #else
pthread_mutex_unlock(&mutex->lock); pthread_mutex_unlock(&mutex->lock);

28
contrib/jemalloc/include/jemalloc/internal/private_namespace.h
View File

@ -7,11 +7,31 @@
#define arena_boot JEMALLOC_N(arena_boot) #define arena_boot JEMALLOC_N(arena_boot)
#define arena_dalloc JEMALLOC_N(arena_dalloc) #define arena_dalloc JEMALLOC_N(arena_dalloc)
#define arena_dalloc_bin JEMALLOC_N(arena_dalloc_bin) #define arena_dalloc_bin JEMALLOC_N(arena_dalloc_bin)
#define arena_dalloc_bin_locked JEMALLOC_N(arena_dalloc_bin_locked)
#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) #define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small)
#define arena_dalloc_large JEMALLOC_N(arena_dalloc_large) #define arena_dalloc_large JEMALLOC_N(arena_dalloc_large)
#define arena_dalloc_large_locked JEMALLOC_N(arena_dalloc_large_locked)
#define arena_dalloc_small JEMALLOC_N(arena_dalloc_small)
#define arena_malloc JEMALLOC_N(arena_malloc) #define arena_malloc JEMALLOC_N(arena_malloc)
#define arena_malloc_large JEMALLOC_N(arena_malloc_large) #define arena_malloc_large JEMALLOC_N(arena_malloc_large)
#define arena_malloc_small JEMALLOC_N(arena_malloc_small) #define arena_malloc_small JEMALLOC_N(arena_malloc_small)
#define arena_mapbits_allocated_get JEMALLOC_N(arena_mapbits_allocated_get)
#define arena_mapbits_binind_get JEMALLOC_N(arena_mapbits_binind_get)
#define arena_mapbits_dirty_get JEMALLOC_N(arena_mapbits_dirty_get)
#define arena_mapbits_get JEMALLOC_N(arena_mapbits_get)
#define arena_mapbits_large_binind_set JEMALLOC_N(arena_mapbits_large_binind_set)
#define arena_mapbits_large_get JEMALLOC_N(arena_mapbits_large_get)
#define arena_mapbits_large_set JEMALLOC_N(arena_mapbits_large_set)
#define arena_mapbits_large_size_get JEMALLOC_N(arena_mapbits_large_size_get)
#define arena_mapbits_small_runind_get JEMALLOC_N(arena_mapbits_small_runind_get)
#define arena_mapbits_small_set JEMALLOC_N(arena_mapbits_small_set)
#define arena_mapbits_unallocated_set JEMALLOC_N(arena_mapbits_unallocated_set)
#define arena_mapbits_unallocated_size_get JEMALLOC_N(arena_mapbits_unallocated_size_get)
#define arena_mapbits_unallocated_size_set JEMALLOC_N(arena_mapbits_unallocated_size_set)
#define arena_mapbits_unzeroed_get JEMALLOC_N(arena_mapbits_unzeroed_get)
#define arena_mapbits_unzeroed_set JEMALLOC_N(arena_mapbits_unzeroed_set)
#define arena_mapbitsp_get JEMALLOC_N(arena_mapbitsp_get)
#define arena_mapp_get JEMALLOC_N(arena_mapp_get)
#define arena_maxclass JEMALLOC_N(arena_maxclass) #define arena_maxclass JEMALLOC_N(arena_maxclass)
#define arena_new JEMALLOC_N(arena_new) #define arena_new JEMALLOC_N(arena_new)
#define arena_palloc JEMALLOC_N(arena_palloc) #define arena_palloc JEMALLOC_N(arena_palloc)
@ -22,6 +42,7 @@
#define arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get) #define arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get)
#define arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set) #define arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set)
#define arena_prof_promoted JEMALLOC_N(arena_prof_promoted) #define arena_prof_promoted JEMALLOC_N(arena_prof_promoted)
#define arena_ptr_small_binind_get JEMALLOC_N(arena_ptr_small_binind_get)
#define arena_purge_all JEMALLOC_N(arena_purge_all) #define arena_purge_all JEMALLOC_N(arena_purge_all)
#define arena_ralloc JEMALLOC_N(arena_ralloc) #define arena_ralloc JEMALLOC_N(arena_ralloc)
#define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move) #define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move)
@ -232,13 +253,13 @@
#define prof_lookup JEMALLOC_N(prof_lookup) #define prof_lookup JEMALLOC_N(prof_lookup)
#define prof_malloc JEMALLOC_N(prof_malloc) #define prof_malloc JEMALLOC_N(prof_malloc)
#define prof_mdump JEMALLOC_N(prof_mdump) #define prof_mdump JEMALLOC_N(prof_mdump)
#define prof_lookup JEMALLOC_N(prof_lookup)
#define prof_promote JEMALLOC_N(prof_promote) #define prof_promote JEMALLOC_N(prof_promote)
#define prof_realloc JEMALLOC_N(prof_realloc) #define prof_realloc JEMALLOC_N(prof_realloc)
#define prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update) #define prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update)
#define prof_sample_threshold_update JEMALLOC_N(prof_sample_threshold_update) #define prof_sample_threshold_update JEMALLOC_N(prof_sample_threshold_update)
#define prof_tdata_booted JEMALLOC_N(prof_tdata_booted) #define prof_tdata_booted JEMALLOC_N(prof_tdata_booted)
#define prof_tdata_cleanup JEMALLOC_N(prof_tdata_cleanup) #define prof_tdata_cleanup JEMALLOC_N(prof_tdata_cleanup)
#define prof_tdata_get JEMALLOC_N(prof_tdata_get)
#define prof_tdata_init JEMALLOC_N(prof_tdata_init) #define prof_tdata_init JEMALLOC_N(prof_tdata_init)
#define prof_tdata_initialized JEMALLOC_N(prof_tdata_initialized) #define prof_tdata_initialized JEMALLOC_N(prof_tdata_initialized)
#define prof_tdata_tls JEMALLOC_N(prof_tdata_tls) #define prof_tdata_tls JEMALLOC_N(prof_tdata_tls)
@ -294,12 +315,13 @@
#define tcache_enabled_tsd_get JEMALLOC_N(tcache_enabled_tsd_get) #define tcache_enabled_tsd_get JEMALLOC_N(tcache_enabled_tsd_get)
#define tcache_enabled_tsd_set JEMALLOC_N(tcache_enabled_tsd_set) #define tcache_enabled_tsd_set JEMALLOC_N(tcache_enabled_tsd_set)
#define tcache_event JEMALLOC_N(tcache_event) #define tcache_event JEMALLOC_N(tcache_event)
#define tcache_initialized JEMALLOC_N(tcache_initialized) #define tcache_event_hard JEMALLOC_N(tcache_event_hard)
#define tcache_flush JEMALLOC_N(tcache_flush) #define tcache_flush JEMALLOC_N(tcache_flush)
#define tcache_get JEMALLOC_N(tcache_get) #define tcache_get JEMALLOC_N(tcache_get)
#define tcache_initialized JEMALLOC_N(tcache_initialized)
#define tcache_maxclass JEMALLOC_N(tcache_maxclass) #define tcache_maxclass JEMALLOC_N(tcache_maxclass)
#define tcache_stats_merge JEMALLOC_N(tcache_stats_merge)
#define tcache_salloc JEMALLOC_N(tcache_salloc) #define tcache_salloc JEMALLOC_N(tcache_salloc)
#define tcache_stats_merge JEMALLOC_N(tcache_stats_merge)
#define tcache_thread_cleanup JEMALLOC_N(tcache_thread_cleanup) #define tcache_thread_cleanup JEMALLOC_N(tcache_thread_cleanup)
#define tcache_tls JEMALLOC_N(tcache_tls) #define tcache_tls JEMALLOC_N(tcache_tls)
#define tcache_tsd_boot JEMALLOC_N(tcache_tsd_boot) #define tcache_tsd_boot JEMALLOC_N(tcache_tsd_boot)

60
contrib/jemalloc/include/jemalloc/internal/prof.h
View File

@ -37,6 +37,14 @@ typedef struct prof_tdata_s prof_tdata_t;
*/ */
#define PROF_NCTX_LOCKS 1024 #define PROF_NCTX_LOCKS 1024
/*
* prof_tdata pointers close to NULL are used to encode state information that
* is used for cleaning up during thread shutdown.
*/
#define PROF_TDATA_STATE_REINCARNATED ((prof_tdata_t *)(uintptr_t)1)
#define PROF_TDATA_STATE_PURGATORY ((prof_tdata_t *)(uintptr_t)2)
#define PROF_TDATA_STATE_MAX PROF_TDATA_STATE_PURGATORY
#endif /* JEMALLOC_H_TYPES */ #endif /* JEMALLOC_H_TYPES */
/******************************************************************************/ /******************************************************************************/
#ifdef JEMALLOC_H_STRUCTS #ifdef JEMALLOC_H_STRUCTS
@ -113,9 +121,19 @@ struct prof_ctx_s {
/* Associated backtrace. */ /* Associated backtrace. */
prof_bt_t *bt; prof_bt_t *bt;
/* Protects cnt_merged and cnts_ql. */ /* Protects nlimbo, cnt_merged, and cnts_ql. */
malloc_mutex_t *lock; malloc_mutex_t *lock;
/*
* Number of threads that currently cause this ctx to be in a state of
* limbo due to one of:
* - Initializing per thread counters associated with this ctx.
* - Preparing to destroy this ctx.
* nlimbo must be 1 (single destroyer) in order to safely destroy the
* ctx.
*/
unsigned nlimbo;
/* Temporary storage for summation during dump. */ /* Temporary storage for summation during dump. */
prof_cnt_t cnt_summed; prof_cnt_t cnt_summed;
@ -152,6 +170,11 @@ struct prof_tdata_s {
uint64_t prng_state; uint64_t prng_state;
uint64_t threshold; uint64_t threshold;
uint64_t accum; uint64_t accum;
/* State used to avoid dumping while operating on prof internals. */
bool enq;
bool enq_idump;
bool enq_gdump;
}; };
#endif /* JEMALLOC_H_STRUCTS */ #endif /* JEMALLOC_H_STRUCTS */
@ -211,13 +234,13 @@ bool prof_boot2(void);
\ \
assert(size == s2u(size)); \ assert(size == s2u(size)); \
\ \
prof_tdata = *prof_tdata_tsd_get(); \ prof_tdata = prof_tdata_get(); \
if (prof_tdata == NULL) { \ if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) { \
prof_tdata = prof_tdata_init(); \ if (prof_tdata != NULL) \
if (prof_tdata == NULL) { \ ret = (prof_thr_cnt_t *)(uintptr_t)1U; \
else \
ret = NULL; \ ret = NULL; \
break; \ break; \
} \
} \ } \
\ \
if (opt_prof_active == false) { \ if (opt_prof_active == false) { \
@ -260,6 +283,7 @@ bool prof_boot2(void);
#ifndef JEMALLOC_ENABLE_INLINE #ifndef JEMALLOC_ENABLE_INLINE
malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *) malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *)
prof_tdata_t *prof_tdata_get(void);
void prof_sample_threshold_update(prof_tdata_t *prof_tdata); void prof_sample_threshold_update(prof_tdata_t *prof_tdata);
prof_ctx_t *prof_ctx_get(const void *ptr); prof_ctx_t *prof_ctx_get(const void *ptr);
void prof_ctx_set(const void *ptr, prof_ctx_t *ctx); void prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
@ -276,6 +300,22 @@ malloc_tsd_externs(prof_tdata, prof_tdata_t *)
malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL, malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL,
prof_tdata_cleanup) prof_tdata_cleanup)
JEMALLOC_INLINE prof_tdata_t *
prof_tdata_get(void)
{
prof_tdata_t *prof_tdata;
cassert(config_prof);
prof_tdata = *prof_tdata_tsd_get();
if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) {
if (prof_tdata == NULL)
prof_tdata = prof_tdata_init();
}
return (prof_tdata);
}
JEMALLOC_INLINE void JEMALLOC_INLINE void
prof_sample_threshold_update(prof_tdata_t *prof_tdata) prof_sample_threshold_update(prof_tdata_t *prof_tdata)
{ {
@ -355,7 +395,8 @@ prof_sample_accum_update(size_t size)
assert(opt_lg_prof_sample != 0); assert(opt_lg_prof_sample != 0);
prof_tdata = *prof_tdata_tsd_get(); prof_tdata = *prof_tdata_tsd_get();
assert(prof_tdata != NULL); if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
return (true);
/* Take care to avoid integer overflow. */ /* Take care to avoid integer overflow. */
if (size >= prof_tdata->threshold - prof_tdata->accum) { if (size >= prof_tdata->threshold - prof_tdata->accum) {
@ -501,8 +542,9 @@ prof_free(const void *ptr, size_t size)
cassert(config_prof); cassert(config_prof);
if ((uintptr_t)ctx > (uintptr_t)1) { if ((uintptr_t)ctx > (uintptr_t)1) {
prof_thr_cnt_t *tcnt;
assert(size == isalloc(ptr, true)); assert(size == isalloc(ptr, true));
prof_thr_cnt_t *tcnt = prof_lookup(ctx->bt); tcnt = prof_lookup(ctx->bt);
if (tcnt != NULL) { if (tcnt != NULL) {
tcnt->epoch++; tcnt->epoch++;

71
contrib/jemalloc/include/jemalloc/internal/tcache.h
View File

@ -101,6 +101,7 @@ extern size_t nhbins;
extern size_t tcache_maxclass; extern size_t tcache_maxclass;
size_t tcache_salloc(const void *ptr); size_t tcache_salloc(const void *ptr);
void tcache_event_hard(tcache_t *tcache);
void *tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, void *tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin,
size_t binind); size_t binind);
void tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, void tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem,
@ -132,7 +133,7 @@ void tcache_enabled_set(bool enabled);
void *tcache_alloc_easy(tcache_bin_t *tbin); void *tcache_alloc_easy(tcache_bin_t *tbin);
void *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero); void *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero);
void *tcache_alloc_large(tcache_t *tcache, size_t size, bool zero); void *tcache_alloc_large(tcache_t *tcache, size_t size, bool zero);
void tcache_dalloc_small(tcache_t *tcache, void *ptr); void tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind);
void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size); void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size);
#endif #endif
@ -266,47 +267,8 @@ tcache_event(tcache_t *tcache)
tcache->ev_cnt++; tcache->ev_cnt++;
assert(tcache->ev_cnt <= TCACHE_GC_INCR); assert(tcache->ev_cnt <= TCACHE_GC_INCR);
if (tcache->ev_cnt == TCACHE_GC_INCR) { if (tcache->ev_cnt == TCACHE_GC_INCR)
size_t binind = tcache->next_gc_bin; tcache_event_hard(tcache);
tcache_bin_t *tbin = &tcache->tbins[binind];
tcache_bin_info_t *tbin_info = &tcache_bin_info[binind];
if (tbin->low_water > 0) {
/*
* Flush (ceiling) 3/4 of the objects below the low
* water mark.
*/
if (binind < NBINS) {
tcache_bin_flush_small(tbin, binind,
tbin->ncached - tbin->low_water +
(tbin->low_water >> 2), tcache);
} else {
tcache_bin_flush_large(tbin, binind,
tbin->ncached - tbin->low_water +
(tbin->low_water >> 2), tcache);
}
/*
* Reduce fill count by 2X. Limit lg_fill_div such that
* the fill count is always at least 1.
*/
if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1))
>= 1)
tbin->lg_fill_div++;
} else if (tbin->low_water < 0) {
/*
* Increase fill count by 2X. Make sure lg_fill_div
* stays greater than 0.
*/
if (tbin->lg_fill_div > 1)
tbin->lg_fill_div--;
}
tbin->low_water = tbin->ncached;
tcache->next_gc_bin++;
if (tcache->next_gc_bin == nhbins)
tcache->next_gc_bin = 0;
tcache->ev_cnt = 0;
}
} }
JEMALLOC_INLINE void * JEMALLOC_INLINE void *
@ -390,13 +352,13 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
if (ret == NULL) if (ret == NULL)
return (NULL); return (NULL);
} else { } else {
if (config_prof) { if (config_prof && prof_promote && size == PAGE) {
arena_chunk_t *chunk = arena_chunk_t *chunk =
(arena_chunk_t *)CHUNK_ADDR2BASE(ret); (arena_chunk_t *)CHUNK_ADDR2BASE(ret);
size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >> size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >>
LG_PAGE); LG_PAGE);
chunk->map[pageind-map_bias].bits &= arena_mapbits_large_binind_set(chunk, pageind,
~CHUNK_MAP_CLASS_MASK; BININD_INVALID);
} }
if (zero == false) { if (zero == false) {
if (config_fill) { if (config_fill) {
@ -421,30 +383,13 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero)
} }
JEMALLOC_INLINE void JEMALLOC_INLINE void
tcache_dalloc_small(tcache_t *tcache, void *ptr) tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind)
{ {
arena_t *arena;
arena_chunk_t *chunk;
arena_run_t *run;
arena_bin_t *bin;
tcache_bin_t *tbin; tcache_bin_t *tbin;
tcache_bin_info_t *tbin_info; tcache_bin_info_t *tbin_info;
size_t pageind, binind;
arena_chunk_map_t *mapelm;
assert(tcache_salloc(ptr) <= SMALL_MAXCLASS); assert(tcache_salloc(ptr) <= SMALL_MAXCLASS);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
arena = chunk->arena;
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
mapelm = &chunk->map[pageind-map_bias];
run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind -
(mapelm->bits >> LG_PAGE)) << LG_PAGE));
bin = run->bin;
binind = ((uintptr_t)bin - (uintptr_t)&arena->bins) /
sizeof(arena_bin_t);
assert(binind < NBINS);
if (config_fill && opt_junk) if (config_fill && opt_junk)
arena_dalloc_junk_small(ptr, &arena_bin_info[binind]); arena_dalloc_junk_small(ptr, &arena_bin_info[binind]);

101
contrib/jemalloc/include/jemalloc/internal/tsd.h
View File

@ -74,6 +74,10 @@ extern bool a_name##_booted;
extern __thread a_type a_name##_tls; \ extern __thread a_type a_name##_tls; \
extern pthread_key_t a_name##_tsd; \ extern pthread_key_t a_name##_tsd; \
extern bool a_name##_booted; extern bool a_name##_booted;
#elif (defined(_WIN32))
#define malloc_tsd_externs(a_name, a_type) \
extern DWORD a_name##_tsd; \
extern bool a_name##_booted;
#else #else
#define malloc_tsd_externs(a_name, a_type) \ #define malloc_tsd_externs(a_name, a_type) \
extern pthread_key_t a_name##_tsd; \ extern pthread_key_t a_name##_tsd; \
@ -94,6 +98,10 @@ a_attr __thread a_type JEMALLOC_TLS_MODEL \
a_name##_tls = a_initializer; \ a_name##_tls = a_initializer; \
a_attr pthread_key_t a_name##_tsd; \ a_attr pthread_key_t a_name##_tsd; \
a_attr bool a_name##_booted = false; a_attr bool a_name##_booted = false;
#elif (defined(_WIN32))
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
a_attr DWORD a_name##_tsd; \
a_attr bool a_name##_booted = false;
#else #else
#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \
a_attr pthread_key_t a_name##_tsd; \ a_attr pthread_key_t a_name##_tsd; \
@ -182,6 +190,99 @@ a_name##_tsd_set(a_type *val) \
} \ } \
} \ } \
} }
#elif (defined(_WIN32))
#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
a_cleanup) \
/* Data structure. */ \
typedef struct { \
bool initialized; \
a_type val; \
} a_name##_tsd_wrapper_t; \
/* Initialization/cleanup. */ \
a_attr bool \
a_name##_tsd_cleanup_wrapper(void) \
{ \
a_name##_tsd_wrapper_t *wrapper; \
\
wrapper = (a_name##_tsd_wrapper_t *) TlsGetValue(a_name##_tsd); \
if (wrapper == NULL) \
return (false); \
if (a_cleanup != malloc_tsd_no_cleanup && \
wrapper->initialized) { \
a_type val = wrapper->val; \
a_type tsd_static_data = a_initializer; \
wrapper->initialized = false; \
wrapper->val = tsd_static_data; \
a_cleanup(&val); \
if (wrapper->initialized) { \
/* Trigger another cleanup round. */ \
return (true); \
} \
} \
malloc_tsd_dalloc(wrapper); \
return (false); \
} \
a_attr bool \
a_name##_tsd_boot(void) \
{ \
\
a_name##_tsd = TlsAlloc(); \
if (a_name##_tsd == TLS_OUT_OF_INDEXES) \
return (true); \
if (a_cleanup != malloc_tsd_no_cleanup) { \
malloc_tsd_cleanup_register( \
&a_name##_tsd_cleanup_wrapper); \
} \
a_name##_booted = true; \
return (false); \
} \
/* Get/set. */ \
a_attr a_name##_tsd_wrapper_t * \
a_name##_tsd_get_wrapper(void) \
{ \
a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \
TlsGetValue(a_name##_tsd); \
\
if (wrapper == NULL) { \
wrapper = (a_name##_tsd_wrapper_t *) \
malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \
if (wrapper == NULL) { \
malloc_write("<jemalloc>: Error allocating" \
" TSD for "#a_name"\n"); \
abort(); \
} else { \
static a_type tsd_static_data = a_initializer; \
wrapper->initialized = false; \
wrapper->val = tsd_static_data; \
} \
if (!TlsSetValue(a_name##_tsd, (void *)wrapper)) { \
malloc_write("<jemalloc>: Error setting" \
" TSD for "#a_name"\n"); \
abort(); \
} \
} \
return (wrapper); \
} \
a_attr a_type * \
a_name##_tsd_get(void) \
{ \
a_name##_tsd_wrapper_t *wrapper; \
\
assert(a_name##_booted); \
wrapper = a_name##_tsd_get_wrapper(); \
return (&wrapper->val); \
} \
a_attr void \
a_name##_tsd_set(a_type *val) \
{ \
a_name##_tsd_wrapper_t *wrapper; \
\
assert(a_name##_booted); \
wrapper = a_name##_tsd_get_wrapper(); \
wrapper->val = *(val); \
if (a_cleanup != malloc_tsd_no_cleanup) \
wrapper->initialized = true; \
}
#else #else
#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \
a_cleanup) \ a_cleanup) \

32
contrib/jemalloc/include/jemalloc/internal/util.h
View File

@ -82,10 +82,9 @@
/******************************************************************************/ /******************************************************************************/
#ifdef JEMALLOC_H_EXTERNS #ifdef JEMALLOC_H_EXTERNS
extern void (*je_malloc_message)(void *wcbopaque, const char *s); int buferror(char *buf, size_t buflen);
int buferror(int errnum, char *buf, size_t buflen);
uintmax_t malloc_strtoumax(const char *nptr, char **endptr, int base); uintmax_t malloc_strtoumax(const char *nptr, char **endptr, int base);
void malloc_write(const char *s);
/* /*
* malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating
@ -109,6 +108,8 @@ void malloc_printf(const char *format, ...)
#ifndef JEMALLOC_ENABLE_INLINE #ifndef JEMALLOC_ENABLE_INLINE
size_t pow2_ceil(size_t x); size_t pow2_ceil(size_t x);
void malloc_write(const char *s); void malloc_write(const char *s);
void set_errno(int errnum);
int get_errno(void);
#endif #endif
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_))
@ -130,15 +131,28 @@ pow2_ceil(size_t x)
return (x); return (x);
} }
/* /* Sets error code */
* Wrapper around malloc_message() that avoids the need for
* je_malloc_message(...) throughout the code.
*/
JEMALLOC_INLINE void JEMALLOC_INLINE void
malloc_write(const char *s) set_errno(int errnum)
{ {
je_malloc_message(NULL, s); #ifdef _WIN32
SetLastError(errnum);
#else
errno = errnum;
#endif
}
/* Get last error code */
JEMALLOC_INLINE int
get_errno(void)
{
#ifdef _WIN32
return (GetLastError());
#else
return (errno);
#endif
} }
#endif #endif

64
contrib/jemalloc/include/jemalloc/jemalloc.h
View File

@ -7,12 +7,12 @@ extern "C" {
#include <limits.h> #include <limits.h>
#include <strings.h> #include <strings.h>
#define JEMALLOC_VERSION "1.0.0-286-ga8f8d7540d66ddee7337db80c92890916e1063ca" #define JEMALLOC_VERSION "1.0.0-335-g37b6f95dcd866f51c91488531a2efc3ed4c2b754"
#define JEMALLOC_VERSION_MAJOR 1 #define JEMALLOC_VERSION_MAJOR 1
#define JEMALLOC_VERSION_MINOR 0 #define JEMALLOC_VERSION_MINOR 0
#define JEMALLOC_VERSION_BUGFIX 0 #define JEMALLOC_VERSION_BUGFIX 0
#define JEMALLOC_VERSION_NREV 286 #define JEMALLOC_VERSION_NREV 335
#define JEMALLOC_VERSION_GID "a8f8d7540d66ddee7337db80c92890916e1063ca" #define JEMALLOC_VERSION_GID "37b6f95dcd866f51c91488531a2efc3ed4c2b754"
#include "jemalloc_defs.h" #include "jemalloc_defs.h"
#include "jemalloc_FreeBSD.h" #include "jemalloc_FreeBSD.h"
@ -37,35 +37,49 @@ extern "C" {
* namespace management, and should be omitted in application code unless * namespace management, and should be omitted in application code unless
* JEMALLOC_NO_DEMANGLE is defined (see below). * JEMALLOC_NO_DEMANGLE is defined (see below).
*/ */
extern const char *je_malloc_conf; extern JEMALLOC_EXPORT const char *je_malloc_conf;
extern void (*je_malloc_message)(void *, const char *); extern JEMALLOC_EXPORT void (*je_malloc_message)(void *cbopaque,
const char *s);
void *je_malloc(size_t size) JEMALLOC_ATTR(malloc); JEMALLOC_EXPORT void *je_malloc(size_t size) JEMALLOC_ATTR(malloc);
void *je_calloc(size_t num, size_t size) JEMALLOC_ATTR(malloc); JEMALLOC_EXPORT void *je_calloc(size_t num, size_t size)
int je_posix_memalign(void **memptr, size_t alignment, size_t size) JEMALLOC_ATTR(malloc);
JEMALLOC_ATTR(nonnull(1)); JEMALLOC_EXPORT int je_posix_memalign(void **memptr, size_t alignment,
void *je_aligned_alloc(size_t alignment, size_t size) JEMALLOC_ATTR(malloc); size_t size) JEMALLOC_ATTR(nonnull(1));
void *je_realloc(void *ptr, size_t size); JEMALLOC_EXPORT void *je_aligned_alloc(size_t alignment, size_t size)
void je_free(void *ptr); JEMALLOC_ATTR(malloc);
JEMALLOC_EXPORT void *je_realloc(void *ptr, size_t size);
JEMALLOC_EXPORT void je_free(void *ptr);
size_t je_malloc_usable_size(const void *ptr); #ifdef JEMALLOC_OVERRIDE_MEMALIGN
void je_malloc_stats_print(void (*write_cb)(void *, const char *), JEMALLOC_EXPORT void * je_memalign(size_t alignment, size_t size)
void *je_cbopaque, const char *opts); JEMALLOC_ATTR(malloc);
int je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, #endif
size_t newlen);
int je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp); #ifdef JEMALLOC_OVERRIDE_VALLOC
int je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, JEMALLOC_EXPORT void * je_valloc(size_t size) JEMALLOC_ATTR(malloc);
#endif
JEMALLOC_EXPORT size_t je_malloc_usable_size(const void *ptr);
JEMALLOC_EXPORT void je_malloc_stats_print(void (*write_cb)(void *,
const char *), void *je_cbopaque, const char *opts);
JEMALLOC_EXPORT int je_mallctl(const char *name, void *oldp,
size_t *oldlenp, void *newp, size_t newlen); size_t *oldlenp, void *newp, size_t newlen);
JEMALLOC_EXPORT int je_mallctlnametomib(const char *name, size_t *mibp,
size_t *miblenp);
JEMALLOC_EXPORT int je_mallctlbymib(const size_t *mib, size_t miblen,
void *oldp, size_t *oldlenp, void *newp, size_t newlen);
#ifdef JEMALLOC_EXPERIMENTAL #ifdef JEMALLOC_EXPERIMENTAL
int je_allocm(void **ptr, size_t *rsize, size_t size, int flags) JEMALLOC_EXPORT int je_allocm(void **ptr, size_t *rsize, size_t size,
JEMALLOC_ATTR(nonnull(1));
int je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra,
int flags) JEMALLOC_ATTR(nonnull(1)); int flags) JEMALLOC_ATTR(nonnull(1));
int je_sallocm(const void *ptr, size_t *rsize, int flags) JEMALLOC_EXPORT int je_rallocm(void **ptr, size_t *rsize, size_t size,
size_t extra, int flags) JEMALLOC_ATTR(nonnull(1));
JEMALLOC_EXPORT int je_sallocm(const void *ptr, size_t *rsize, int flags)
JEMALLOC_ATTR(nonnull(1)); JEMALLOC_ATTR(nonnull(1));
int je_dallocm(void *ptr, int flags) JEMALLOC_ATTR(nonnull(1)); JEMALLOC_EXPORT int je_dallocm(void *ptr, int flags)
int je_nallocm(size_t *rsize, size_t size, int flags); JEMALLOC_ATTR(nonnull(1));
JEMALLOC_EXPORT int je_nallocm(size_t *rsize, size_t size, int flags);
#endif #endif
/* /*

39
contrib/jemalloc/include/jemalloc/jemalloc_defs.h
View File

@ -105,11 +105,27 @@
/* Defined if __attribute__((...)) syntax is supported. */ /* Defined if __attribute__((...)) syntax is supported. */
#define JEMALLOC_HAVE_ATTR #define JEMALLOC_HAVE_ATTR
#ifdef JEMALLOC_HAVE_ATTR #ifdef JEMALLOC_HAVE_ATTR
# define JEMALLOC_CATTR(s, a) __attribute__((s)) # define JEMALLOC_ATTR(s) __attribute__((s))
# define JEMALLOC_ATTR(s) JEMALLOC_CATTR(s,) # define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default"))
# define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s))
# define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s))
# define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline)
#elif _MSC_VER
# define JEMALLOC_ATTR(s)
# ifdef DLLEXPORT
# define JEMALLOC_EXPORT __declspec(dllexport)
# else
# define JEMALLOC_EXPORT __declspec(dllimport)
# endif
# define JEMALLOC_ALIGNED(s) __declspec(align(s))
# define JEMALLOC_SECTION(s) __declspec(allocate(s))
# define JEMALLOC_NOINLINE __declspec(noinline)
#else #else
# define JEMALLOC_CATTR(s, a) a # define JEMALLOC_ATTR(s)
# define JEMALLOC_ATTR(s) JEMALLOC_CATTR(s,) # define JEMALLOC_EXPORT
# define JEMALLOC_ALIGNED(s)
# define JEMALLOC_SECTION(s)
# define JEMALLOC_NOINLINE
#endif #endif
/* Defined if sbrk() is supported. */ /* Defined if sbrk() is supported. */
@ -178,12 +194,18 @@
/* /*
* If defined, use munmap() to unmap freed chunks, rather than storing them for * If defined, use munmap() to unmap freed chunks, rather than storing them for
* later reuse. This is automatically disabled if configuration determines * later reuse. This is disabled by default on Linux because common sequences
* that common sequences of mmap()/munmap() calls will cause virtual memory map * of mmap()/munmap() calls will cause virtual memory map holes.
* holes.
*/ */
#define JEMALLOC_MUNMAP #define JEMALLOC_MUNMAP
/*
* If defined, use mremap(...MREMAP_FIXED...) for huge realloc(). This is
* disabled by default because it is Linux-specific and it will cause virtual
* memory map holes, much like munmap(2) does.
*/
/* #undef JEMALLOC_MREMAP */
/* TLS is used to map arenas and magazine caches to threads. */ /* TLS is used to map arenas and magazine caches to threads. */
#define JEMALLOC_TLS #define JEMALLOC_TLS
@ -206,9 +228,6 @@
/* #undef JEMALLOC_ZONE */ /* #undef JEMALLOC_ZONE */
/* #undef JEMALLOC_ZONE_VERSION */ /* #undef JEMALLOC_ZONE_VERSION */
/* If defined, use mremap(...MREMAP_FIXED...) for huge realloc(). */
/* #undef JEMALLOC_MREMAP_FIXED */
/* /*
* Methods for purging unused pages differ between operating systems. * Methods for purging unused pages differ between operating systems.
* *

503
contrib/jemalloc/src/arena.c
View File

@ -7,7 +7,7 @@
ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT;
arena_bin_info_t arena_bin_info[NBINS]; arena_bin_info_t arena_bin_info[NBINS];
JEMALLOC_ATTR(aligned(CACHELINE)) JEMALLOC_ALIGNED(CACHELINE)
const uint8_t small_size2bin[] = { const uint8_t small_size2bin[] = {
#define S2B_8(i) i, #define S2B_8(i) i,
#define S2B_16(i) S2B_8(i) S2B_8(i) #define S2B_16(i) S2B_8(i) S2B_8(i)
@ -41,11 +41,11 @@ const uint8_t small_size2bin[] = {
/* Function prototypes for non-inline static functions. */ /* Function prototypes for non-inline static functions. */
static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size,
bool large, bool zero); bool large, size_t binind, bool zero);
static arena_chunk_t *arena_chunk_alloc(arena_t *arena); static arena_chunk_t *arena_chunk_alloc(arena_t *arena);
static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk);
static arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, static arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large,
bool zero); size_t binind, bool zero);
static void arena_purge(arena_t *arena, bool all); static void arena_purge(arena_t *arena, bool all);
static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty); static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty);
static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk,
@ -152,7 +152,9 @@ static inline void
arena_run_reg_dalloc(arena_run_t *run, void *ptr) arena_run_reg_dalloc(arena_run_t *run, void *ptr)
{ {
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
size_t binind = arena_bin_index(chunk->arena, run->bin); size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
size_t mapbits = arena_mapbits_get(chunk, pageind);
size_t binind = arena_ptr_small_binind_get(ptr, mapbits);
arena_bin_info_t *bin_info = &arena_bin_info[binind]; arena_bin_info_t *bin_info = &arena_bin_info[binind];
unsigned regind = arena_run_regind(run, bin_info, ptr); unsigned regind = arena_run_regind(run, bin_info, ptr);
bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run +
@ -184,28 +186,31 @@ arena_chunk_validate_zeroed(arena_chunk_t *chunk, size_t run_ind)
static void static void
arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
bool zero) size_t binind, bool zero)
{ {
arena_chunk_t *chunk; arena_chunk_t *chunk;
size_t run_ind, total_pages, need_pages, rem_pages, i; size_t run_ind, total_pages, need_pages, rem_pages, i;
size_t flag_dirty; size_t flag_dirty;
arena_avail_tree_t *runs_avail; arena_avail_tree_t *runs_avail;
assert((large && binind == BININD_INVALID) || (large == false && binind
!= BININD_INVALID));
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE);
flag_dirty = chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY; flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);
runs_avail = (flag_dirty != 0) ? &arena->runs_avail_dirty : runs_avail = (flag_dirty != 0) ? &arena->runs_avail_dirty :
&arena->runs_avail_clean; &arena->runs_avail_clean;
total_pages = (chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) >> total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >>
LG_PAGE; LG_PAGE;
assert((chunk->map[run_ind+total_pages-1-map_bias].bits & assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) ==
CHUNK_MAP_DIRTY) == flag_dirty); flag_dirty);
need_pages = (size >> LG_PAGE); need_pages = (size >> LG_PAGE);
assert(need_pages > 0); assert(need_pages > 0);
assert(need_pages <= total_pages); assert(need_pages <= total_pages);
rem_pages = total_pages - need_pages; rem_pages = total_pages - need_pages;
arena_avail_tree_remove(runs_avail, &chunk->map[run_ind-map_bias]); arena_avail_tree_remove(runs_avail, arena_mapp_get(chunk, run_ind));
if (config_stats) { if (config_stats) {
/* /*
* Update stats_cactive if nactive is crossing a chunk * Update stats_cactive if nactive is crossing a chunk
@ -222,22 +227,23 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
/* Keep track of trailing unused pages for later use. */ /* Keep track of trailing unused pages for later use. */
if (rem_pages > 0) { if (rem_pages > 0) {
if (flag_dirty != 0) { if (flag_dirty != 0) {
chunk->map[run_ind+need_pages-map_bias].bits = arena_mapbits_unallocated_set(chunk, run_ind+need_pages,
(rem_pages << LG_PAGE) | CHUNK_MAP_DIRTY; (rem_pages << LG_PAGE), CHUNK_MAP_DIRTY);
chunk->map[run_ind+total_pages-1-map_bias].bits = arena_mapbits_unallocated_set(chunk,
(rem_pages << LG_PAGE) | CHUNK_MAP_DIRTY; run_ind+total_pages-1, (rem_pages << LG_PAGE),
CHUNK_MAP_DIRTY);
} else { } else {
chunk->map[run_ind+need_pages-map_bias].bits = arena_mapbits_unallocated_set(chunk, run_ind+need_pages,
(rem_pages << LG_PAGE) | (rem_pages << LG_PAGE),
(chunk->map[run_ind+need_pages-map_bias].bits & arena_mapbits_unzeroed_get(chunk,
CHUNK_MAP_UNZEROED); run_ind+need_pages));
chunk->map[run_ind+total_pages-1-map_bias].bits = arena_mapbits_unallocated_set(chunk,
(rem_pages << LG_PAGE) | run_ind+total_pages-1, (rem_pages << LG_PAGE),
(chunk->map[run_ind+total_pages-1-map_bias].bits & arena_mapbits_unzeroed_get(chunk,
CHUNK_MAP_UNZEROED); run_ind+total_pages-1));
} }
arena_avail_tree_insert(runs_avail, arena_avail_tree_insert(runs_avail, arena_mapp_get(chunk,
&chunk->map[run_ind+need_pages-map_bias]); run_ind+need_pages));
} }
/* Update dirty page accounting. */ /* Update dirty page accounting. */
@ -258,8 +264,8 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
* zeroed (i.e. never before touched). * zeroed (i.e. never before touched).
*/ */
for (i = 0; i < need_pages; i++) { for (i = 0; i < need_pages; i++) {
if ((chunk->map[run_ind+i-map_bias].bits if (arena_mapbits_unzeroed_get(chunk,
& CHUNK_MAP_UNZEROED) != 0) { run_ind+i) != 0) {
VALGRIND_MAKE_MEM_UNDEFINED( VALGRIND_MAKE_MEM_UNDEFINED(
(void *)((uintptr_t) (void *)((uintptr_t)
chunk + ((run_ind+i) << chunk + ((run_ind+i) <<
@ -293,10 +299,9 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
* Set the last element first, in case the run only contains one * Set the last element first, in case the run only contains one
* page (i.e. both statements set the same element). * page (i.e. both statements set the same element).
*/ */
chunk->map[run_ind+need_pages-1-map_bias].bits = arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0,
CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED | flag_dirty; flag_dirty);
chunk->map[run_ind-map_bias].bits = size | flag_dirty | arena_mapbits_large_set(chunk, run_ind, size, flag_dirty);
CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED;
} else { } else {
assert(zero == false); assert(zero == false);
/* /*
@ -304,34 +309,30 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large,
* small run, so that arena_dalloc_bin_run() has the ability to * small run, so that arena_dalloc_bin_run() has the ability to
* conditionally trim clean pages. * conditionally trim clean pages.
*/ */
chunk->map[run_ind-map_bias].bits = arena_mapbits_small_set(chunk, run_ind, 0, binind,
(chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) | arena_mapbits_unzeroed_get(chunk, run_ind) | flag_dirty);
CHUNK_MAP_ALLOCATED | flag_dirty;
/* /*
* The first page will always be dirtied during small run * The first page will always be dirtied during small run
* initialization, so a validation failure here would not * initialization, so a validation failure here would not
* actually cause an observable failure. * actually cause an observable failure.
*/ */
if (config_debug && flag_dirty == 0 && if (config_debug && flag_dirty == 0 &&
(chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) arena_mapbits_unzeroed_get(chunk, run_ind) == 0)
== 0)
arena_chunk_validate_zeroed(chunk, run_ind); arena_chunk_validate_zeroed(chunk, run_ind);
for (i = 1; i < need_pages - 1; i++) { for (i = 1; i < need_pages - 1; i++) {
chunk->map[run_ind+i-map_bias].bits = (i << LG_PAGE) arena_mapbits_small_set(chunk, run_ind+i, i,
| (chunk->map[run_ind+i-map_bias].bits & binind, arena_mapbits_unzeroed_get(chunk,
CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED; run_ind+i));
if (config_debug && flag_dirty == 0 && if (config_debug && flag_dirty == 0 &&
(chunk->map[run_ind+i-map_bias].bits & arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0)
CHUNK_MAP_UNZEROED) == 0)
arena_chunk_validate_zeroed(chunk, run_ind+i); arena_chunk_validate_zeroed(chunk, run_ind+i);
} }
chunk->map[run_ind+need_pages-1-map_bias].bits = ((need_pages arena_mapbits_small_set(chunk, run_ind+need_pages-1,
- 1) << LG_PAGE) | need_pages-1, binind, arena_mapbits_unzeroed_get(chunk,
(chunk->map[run_ind+need_pages-1-map_bias].bits & run_ind+need_pages-1) | flag_dirty);
CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED | flag_dirty;
if (config_debug && flag_dirty == 0 && if (config_debug && flag_dirty == 0 &&
(chunk->map[run_ind+need_pages-1-map_bias].bits & arena_mapbits_unzeroed_get(chunk, run_ind+need_pages-1) ==
CHUNK_MAP_UNZEROED) == 0) { 0) {
arena_chunk_validate_zeroed(chunk, arena_chunk_validate_zeroed(chunk,
run_ind+need_pages-1); run_ind+need_pages-1);
} }
@ -351,17 +352,18 @@ arena_chunk_alloc(arena_t *arena)
arena->spare = NULL; arena->spare = NULL;
/* Insert the run into the appropriate runs_avail_* tree. */ /* Insert the run into the appropriate runs_avail_* tree. */
if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) if (arena_mapbits_dirty_get(chunk, map_bias) == 0)
runs_avail = &arena->runs_avail_clean; runs_avail = &arena->runs_avail_clean;
else else
runs_avail = &arena->runs_avail_dirty; runs_avail = &arena->runs_avail_dirty;
assert((chunk->map[0].bits & ~PAGE_MASK) == arena_maxclass); assert(arena_mapbits_unallocated_size_get(chunk, map_bias) ==
assert((chunk->map[chunk_npages-1-map_bias].bits & ~PAGE_MASK) arena_maxclass);
== arena_maxclass); assert(arena_mapbits_unallocated_size_get(chunk,
assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) == chunk_npages-1) == arena_maxclass);
(chunk->map[chunk_npages-1-map_bias].bits & assert(arena_mapbits_dirty_get(chunk, map_bias) ==
CHUNK_MAP_DIRTY)); arena_mapbits_dirty_get(chunk, chunk_npages-1));
arena_avail_tree_insert(runs_avail, &chunk->map[0]); arena_avail_tree_insert(runs_avail, arena_mapp_get(chunk,
map_bias));
} else { } else {
bool zero; bool zero;
size_t unzeroed; size_t unzeroed;
@ -392,24 +394,27 @@ arena_chunk_alloc(arena_t *arena)
* chunk. * chunk.
*/ */
unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED;
chunk->map[0].bits = arena_maxclass | unzeroed; arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass,
unzeroed);
/* /*
* There is no need to initialize the internal page map entries * There is no need to initialize the internal page map entries
* unless the chunk is not zeroed. * unless the chunk is not zeroed.
*/ */
if (zero == false) { if (zero == false) {
for (i = map_bias+1; i < chunk_npages-1; i++) for (i = map_bias+1; i < chunk_npages-1; i++)
chunk->map[i-map_bias].bits = unzeroed; arena_mapbits_unzeroed_set(chunk, i, unzeroed);
} else if (config_debug) { } else if (config_debug) {
for (i = map_bias+1; i < chunk_npages-1; i++) for (i = map_bias+1; i < chunk_npages-1; i++) {
assert(chunk->map[i-map_bias].bits == unzeroed); assert(arena_mapbits_unzeroed_get(chunk, i) ==
unzeroed);
}
} }
chunk->map[chunk_npages-1-map_bias].bits = arena_maxclass | arena_mapbits_unallocated_set(chunk, chunk_npages-1,
unzeroed; arena_maxclass, unzeroed);
/* Insert the run into the runs_avail_clean tree. */ /* Insert the run into the runs_avail_clean tree. */
arena_avail_tree_insert(&arena->runs_avail_clean, arena_avail_tree_insert(&arena->runs_avail_clean,
&chunk->map[0]); arena_mapp_get(chunk, map_bias));
} }
return (chunk); return (chunk);
@ -424,11 +429,11 @@ arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk)
* Remove run from the appropriate runs_avail_* tree, so that the arena * Remove run from the appropriate runs_avail_* tree, so that the arena
* does not use it. * does not use it.
*/ */
if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) if (arena_mapbits_dirty_get(chunk, map_bias) == 0)
runs_avail = &arena->runs_avail_clean; runs_avail = &arena->runs_avail_clean;
else else
runs_avail = &arena->runs_avail_dirty; runs_avail = &arena->runs_avail_dirty;
arena_avail_tree_remove(runs_avail, &chunk->map[0]); arena_avail_tree_remove(runs_avail, arena_mapp_get(chunk, map_bias));
if (arena->spare != NULL) { if (arena->spare != NULL) {
arena_chunk_t *spare = arena->spare; arena_chunk_t *spare = arena->spare;
@ -449,7 +454,8 @@ arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk)
} }
static arena_run_t * static arena_run_t *
arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) arena_run_alloc(arena_t *arena, size_t size, bool large, size_t binind,
bool zero)
{ {
arena_chunk_t *chunk; arena_chunk_t *chunk;
arena_run_t *run; arena_run_t *run;
@ -457,6 +463,8 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero)
assert(size <= arena_maxclass); assert(size <= arena_maxclass);
assert((size & PAGE_MASK) == 0); assert((size & PAGE_MASK) == 0);
assert((large && binind == BININD_INVALID) || (large == false && binind
!= BININD_INVALID));
/* Search the arena's chunks for the lowest best fit. */ /* Search the arena's chunks for the lowest best fit. */
key.bits = size | CHUNK_MAP_KEY; key.bits = size | CHUNK_MAP_KEY;
@ -469,7 +477,7 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero)
run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << run = (arena_run_t *)((uintptr_t)run_chunk + (pageind <<
LG_PAGE)); LG_PAGE));
arena_run_split(arena, run, size, large, zero); arena_run_split(arena, run, size, large, binind, zero);
return (run); return (run);
} }
mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key);
@ -481,7 +489,7 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero)
run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << run = (arena_run_t *)((uintptr_t)run_chunk + (pageind <<
LG_PAGE)); LG_PAGE));
arena_run_split(arena, run, size, large, zero); arena_run_split(arena, run, size, large, binind, zero);
return (run); return (run);
} }
@ -491,7 +499,7 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero)
chunk = arena_chunk_alloc(arena); chunk = arena_chunk_alloc(arena);
if (chunk != NULL) { if (chunk != NULL) {
run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE));
arena_run_split(arena, run, size, large, zero); arena_run_split(arena, run, size, large, binind, zero);
return (run); return (run);
} }
@ -509,7 +517,7 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero)
run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << run = (arena_run_t *)((uintptr_t)run_chunk + (pageind <<
LG_PAGE)); LG_PAGE));
arena_run_split(arena, run, size, large, zero); arena_run_split(arena, run, size, large, binind, zero);
return (run); return (run);
} }
mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key);
@ -521,7 +529,7 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero)
run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << run = (arena_run_t *)((uintptr_t)run_chunk + (pageind <<
LG_PAGE)); LG_PAGE));
arena_run_split(arena, run, size, large, zero); arena_run_split(arena, run, size, large, binind, zero);
return (run); return (run);
} }
@ -579,40 +587,38 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk)
* run. * run.
*/ */
if (chunk == arena->spare) { if (chunk == arena->spare) {
assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) != 0); assert(arena_mapbits_dirty_get(chunk, map_bias) != 0);
arena_chunk_alloc(arena); arena_chunk_alloc(arena);
} }
/* Temporarily allocate all free dirty runs within chunk. */ /* Temporarily allocate all free dirty runs within chunk. */
for (pageind = map_bias; pageind < chunk_npages;) { for (pageind = map_bias; pageind < chunk_npages;) {
mapelm = &chunk->map[pageind-map_bias]; mapelm = arena_mapp_get(chunk, pageind);
if ((mapelm->bits & CHUNK_MAP_ALLOCATED) == 0) { if (arena_mapbits_allocated_get(chunk, pageind) == 0) {
size_t npages; size_t npages;
npages = mapelm->bits >> LG_PAGE; npages = arena_mapbits_unallocated_size_get(chunk,
pageind) >> LG_PAGE;
assert(pageind + npages <= chunk_npages); assert(pageind + npages <= chunk_npages);
if (mapelm->bits & CHUNK_MAP_DIRTY) { if (arena_mapbits_dirty_get(chunk, pageind)) {
size_t i; size_t i;
arena_avail_tree_remove( arena_avail_tree_remove(
&arena->runs_avail_dirty, mapelm); &arena->runs_avail_dirty, mapelm);
mapelm->bits = (npages << LG_PAGE) | arena_mapbits_large_set(chunk, pageind,
flag_unzeroed | CHUNK_MAP_LARGE | (npages << LG_PAGE), flag_unzeroed);
CHUNK_MAP_ALLOCATED;
/* /*
* Update internal elements in the page map, so * Update internal elements in the page map, so
* that CHUNK_MAP_UNZEROED is properly set. * that CHUNK_MAP_UNZEROED is properly set.
*/ */
for (i = 1; i < npages - 1; i++) { for (i = 1; i < npages - 1; i++) {
chunk->map[pageind+i-map_bias].bits = arena_mapbits_unzeroed_set(chunk,
flag_unzeroed; pageind+i, flag_unzeroed);
} }
if (npages > 1) { if (npages > 1) {
chunk->map[ arena_mapbits_large_set(chunk,
pageind+npages-1-map_bias].bits = pageind+npages-1, 0, flag_unzeroed);
flag_unzeroed | CHUNK_MAP_LARGE |
CHUNK_MAP_ALLOCATED;
} }
if (config_stats) { if (config_stats) {
@ -637,17 +643,19 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk)
pageind += npages; pageind += npages;
} else { } else {
/* Skip allocated run. */ /* Skip allocated run. */
if (mapelm->bits & CHUNK_MAP_LARGE) if (arena_mapbits_large_get(chunk, pageind))
pageind += mapelm->bits >> LG_PAGE; pageind += arena_mapbits_large_size_get(chunk,
pageind) >> LG_PAGE;
else { else {
size_t binind;
arena_bin_info_t *bin_info;
arena_run_t *run = (arena_run_t *)((uintptr_t) arena_run_t *run = (arena_run_t *)((uintptr_t)
chunk + (uintptr_t)(pageind << LG_PAGE)); chunk + (uintptr_t)(pageind << LG_PAGE));
assert((mapelm->bits >> LG_PAGE) == 0); assert(arena_mapbits_small_runind_get(chunk,
size_t binind = arena_bin_index(arena, pageind) == 0);
run->bin); binind = arena_bin_index(arena, run->bin);
arena_bin_info_t *bin_info = bin_info = &arena_bin_info[binind];
&arena_bin_info[binind];
pageind += bin_info->run_size >> LG_PAGE; pageind += bin_info->run_size >> LG_PAGE;
} }
} }
@ -669,7 +677,8 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk)
ql_foreach(mapelm, &mapelms, u.ql_link) { ql_foreach(mapelm, &mapelms, u.ql_link) {
size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) /
sizeof(arena_chunk_map_t)) + map_bias; sizeof(arena_chunk_map_t)) + map_bias;
size_t npages = mapelm->bits >> LG_PAGE; size_t npages = arena_mapbits_large_size_get(chunk, pageind) >>
LG_PAGE;
assert(pageind + npages <= chunk_npages); assert(pageind + npages <= chunk_npages);
assert(ndirty >= npages); assert(ndirty >= npages);
@ -806,15 +815,11 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE);
assert(run_ind >= map_bias); assert(run_ind >= map_bias);
assert(run_ind < chunk_npages); assert(run_ind < chunk_npages);
if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_LARGE) != 0) { if (arena_mapbits_large_get(chunk, run_ind) != 0) {
size = chunk->map[run_ind-map_bias].bits & ~PAGE_MASK; size = arena_mapbits_large_size_get(chunk, run_ind);
assert(size == PAGE || assert(size == PAGE ||
(chunk->map[run_ind+(size>>LG_PAGE)-1-map_bias].bits & arena_mapbits_large_size_get(chunk,
~PAGE_MASK) == 0); run_ind+(size>>LG_PAGE)-1) == 0);
assert((chunk->map[run_ind+(size>>LG_PAGE)-1-map_bias].bits &
CHUNK_MAP_LARGE) != 0);
assert((chunk->map[run_ind+(size>>LG_PAGE)-1-map_bias].bits &
CHUNK_MAP_ALLOCATED) != 0);
} else { } else {
size_t binind = arena_bin_index(arena, run->bin); size_t binind = arena_bin_index(arena, run->bin);
arena_bin_info_t *bin_info = &arena_bin_info[binind]; arena_bin_info_t *bin_info = &arena_bin_info[binind];
@ -837,7 +842,7 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
* The run is dirty if the caller claims to have dirtied it, as well as * The run is dirty if the caller claims to have dirtied it, as well as
* if it was already dirty before being allocated. * if it was already dirty before being allocated.
*/ */
if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) != 0) if (arena_mapbits_dirty_get(chunk, run_ind) != 0)
dirty = true; dirty = true;
flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0;
runs_avail = dirty ? &arena->runs_avail_dirty : runs_avail = dirty ? &arena->runs_avail_dirty :
@ -845,58 +850,52 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
/* Mark pages as unallocated in the chunk map. */ /* Mark pages as unallocated in the chunk map. */
if (dirty) { if (dirty) {
chunk->map[run_ind-map_bias].bits = size | CHUNK_MAP_DIRTY; arena_mapbits_unallocated_set(chunk, run_ind, size,
chunk->map[run_ind+run_pages-1-map_bias].bits = size | CHUNK_MAP_DIRTY);
CHUNK_MAP_DIRTY; arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size,
CHUNK_MAP_DIRTY);
chunk->ndirty += run_pages; chunk->ndirty += run_pages;
arena->ndirty += run_pages; arena->ndirty += run_pages;
} else { } else {
chunk->map[run_ind-map_bias].bits = size | arena_mapbits_unallocated_set(chunk, run_ind, size,
(chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED); arena_mapbits_unzeroed_get(chunk, run_ind));
chunk->map[run_ind+run_pages-1-map_bias].bits = size | arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size,
(chunk->map[run_ind+run_pages-1-map_bias].bits & arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1));
CHUNK_MAP_UNZEROED);
} }
/* Try to coalesce forward. */ /* Try to coalesce forward. */
if (run_ind + run_pages < chunk_npages && if (run_ind + run_pages < chunk_npages &&
(chunk->map[run_ind+run_pages-map_bias].bits & CHUNK_MAP_ALLOCATED) arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 &&
== 0 && (chunk->map[run_ind+run_pages-map_bias].bits & arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) {
CHUNK_MAP_DIRTY) == flag_dirty) { size_t nrun_size = arena_mapbits_unallocated_size_get(chunk,
size_t nrun_size = chunk->map[run_ind+run_pages-map_bias].bits & run_ind+run_pages);
~PAGE_MASK;
size_t nrun_pages = nrun_size >> LG_PAGE; size_t nrun_pages = nrun_size >> LG_PAGE;
/* /*
* Remove successor from runs_avail; the coalesced run is * Remove successor from runs_avail; the coalesced run is
* inserted later. * inserted later.
*/ */
assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits assert(arena_mapbits_unallocated_size_get(chunk,
& ~PAGE_MASK) == nrun_size); run_ind+run_pages+nrun_pages-1) == nrun_size);
assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits assert(arena_mapbits_dirty_get(chunk,
& CHUNK_MAP_ALLOCATED) == 0); run_ind+run_pages+nrun_pages-1) == flag_dirty);
assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits
& CHUNK_MAP_DIRTY) == flag_dirty);
arena_avail_tree_remove(runs_avail, arena_avail_tree_remove(runs_avail,
&chunk->map[run_ind+run_pages-map_bias]); arena_mapp_get(chunk, run_ind+run_pages));
size += nrun_size; size += nrun_size;
run_pages += nrun_pages; run_pages += nrun_pages;
chunk->map[run_ind-map_bias].bits = size | arena_mapbits_unallocated_size_set(chunk, run_ind, size);
(chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1,
chunk->map[run_ind+run_pages-1-map_bias].bits = size | size);
(chunk->map[run_ind+run_pages-1-map_bias].bits &
CHUNK_MAP_FLAGS_MASK);
} }
/* Try to coalesce backward. */ /* Try to coalesce backward. */
if (run_ind > map_bias && (chunk->map[run_ind-1-map_bias].bits & if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, run_ind-1)
CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[run_ind-1-map_bias].bits & == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == flag_dirty) {
CHUNK_MAP_DIRTY) == flag_dirty) { size_t prun_size = arena_mapbits_unallocated_size_get(chunk,
size_t prun_size = chunk->map[run_ind-1-map_bias].bits & run_ind-1);
~PAGE_MASK;
size_t prun_pages = prun_size >> LG_PAGE; size_t prun_pages = prun_size >> LG_PAGE;
run_ind -= prun_pages; run_ind -= prun_pages;
@ -905,31 +904,26 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
* Remove predecessor from runs_avail; the coalesced run is * Remove predecessor from runs_avail; the coalesced run is
* inserted later. * inserted later.
*/ */
assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) assert(arena_mapbits_unallocated_size_get(chunk, run_ind) ==
== prun_size); prun_size);
assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_ALLOCATED) assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty);
== 0); arena_avail_tree_remove(runs_avail, arena_mapp_get(chunk,
assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) run_ind));
== flag_dirty);
arena_avail_tree_remove(runs_avail,
&chunk->map[run_ind-map_bias]);
size += prun_size; size += prun_size;
run_pages += prun_pages; run_pages += prun_pages;
chunk->map[run_ind-map_bias].bits = size | arena_mapbits_unallocated_size_set(chunk, run_ind, size);
(chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1,
chunk->map[run_ind+run_pages-1-map_bias].bits = size | size);
(chunk->map[run_ind+run_pages-1-map_bias].bits &
CHUNK_MAP_FLAGS_MASK);
} }
/* Insert into runs_avail, now that coalescing is complete. */ /* Insert into runs_avail, now that coalescing is complete. */
assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) == assert(arena_mapbits_unallocated_size_get(chunk, run_ind) ==
(chunk->map[run_ind+run_pages-1-map_bias].bits & ~PAGE_MASK)); arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1));
assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == assert(arena_mapbits_dirty_get(chunk, run_ind) ==
(chunk->map[run_ind+run_pages-1-map_bias].bits & CHUNK_MAP_DIRTY)); arena_mapbits_dirty_get(chunk, run_ind+run_pages-1));
arena_avail_tree_insert(runs_avail, &chunk->map[run_ind-map_bias]); arena_avail_tree_insert(runs_avail, arena_mapp_get(chunk, run_ind));
if (dirty) { if (dirty) {
/* /*
@ -943,14 +937,15 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty)
} }
} }
/* /* Deallocate chunk if it is now completely unused. */
* Deallocate chunk if it is now completely unused. The bit if (size == arena_maxclass) {
* manipulation checks whether the first run is unallocated and extends assert(run_ind == map_bias);
* to the end of the chunk. assert(run_pages == (arena_maxclass >> LG_PAGE));
*/ assert(arena_mapbits_allocated_get(chunk, map_bias) == 0);
if ((chunk->map[0].bits & (~PAGE_MASK | CHUNK_MAP_ALLOCATED)) == assert(arena_mapbits_unallocated_size_get(chunk, map_bias) ==
arena_maxclass) arena_maxclass);
arena_chunk_dealloc(arena, chunk); arena_chunk_dealloc(arena, chunk);
}
/* /*
* It is okay to do dirty page processing here even if the chunk was * It is okay to do dirty page processing here even if the chunk was
@ -969,7 +964,7 @@ arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
{ {
size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE;
size_t head_npages = (oldsize - newsize) >> LG_PAGE; size_t head_npages = (oldsize - newsize) >> LG_PAGE;
size_t flag_dirty = chunk->map[pageind-map_bias].bits & CHUNK_MAP_DIRTY; size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind);
assert(oldsize > newsize); assert(oldsize > newsize);
@ -978,29 +973,21 @@ arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
* leading run as separately allocated. Set the last element of each * leading run as separately allocated. Set the last element of each
* run first, in case of single-page runs. * run first, in case of single-page runs.
*/ */
assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize);
assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty |
chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | arena_mapbits_unzeroed_get(chunk, pageind+head_npages-1));
(chunk->map[pageind+head_npages-1-map_bias].bits & arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty |
CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; arena_mapbits_unzeroed_get(chunk, pageind));
chunk->map[pageind-map_bias].bits = (oldsize - newsize)
| flag_dirty | (chunk->map[pageind-map_bias].bits &
CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED;
if (config_debug) { if (config_debug) {
UNUSED size_t tail_npages = newsize >> LG_PAGE; UNUSED size_t tail_npages = newsize >> LG_PAGE;
assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] assert(arena_mapbits_large_size_get(chunk,
.bits & ~PAGE_MASK) == 0); pageind+head_npages+tail_npages-1) == 0);
assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] assert(arena_mapbits_dirty_get(chunk,
.bits & CHUNK_MAP_DIRTY) == flag_dirty); pageind+head_npages+tail_npages-1) == flag_dirty);
assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias]
.bits & CHUNK_MAP_LARGE) != 0);
assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias]
.bits & CHUNK_MAP_ALLOCATED) != 0);
} }
chunk->map[pageind+head_npages-map_bias].bits = newsize | flag_dirty | arena_mapbits_large_set(chunk, pageind+head_npages, newsize, flag_dirty
(chunk->map[pageind+head_npages-map_bias].bits & | arena_mapbits_unzeroed_get(chunk, pageind+head_npages));
CHUNK_MAP_FLAGS_MASK) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED;
arena_run_dalloc(arena, run, false); arena_run_dalloc(arena, run, false);
} }
@ -1011,9 +998,7 @@ arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
{ {
size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE;
size_t head_npages = newsize >> LG_PAGE; size_t head_npages = newsize >> LG_PAGE;
size_t tail_npages = (oldsize - newsize) >> LG_PAGE; size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind);
size_t flag_dirty = chunk->map[pageind-map_bias].bits &
CHUNK_MAP_DIRTY;
assert(oldsize > newsize); assert(oldsize > newsize);
@ -1022,28 +1007,22 @@ arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
* trailing run as separately allocated. Set the last element of each * trailing run as separately allocated. Set the last element of each
* run first, in case of single-page runs. * run first, in case of single-page runs.
*/ */
assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize);
assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty |
chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | arena_mapbits_unzeroed_get(chunk, pageind+head_npages-1));
(chunk->map[pageind+head_npages-1-map_bias].bits & arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty |
CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; arena_mapbits_unzeroed_get(chunk, pageind));
chunk->map[pageind-map_bias].bits = newsize | flag_dirty |
(chunk->map[pageind-map_bias].bits & CHUNK_MAP_UNZEROED) |
CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED;
assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & if (config_debug) {
~PAGE_MASK) == 0); UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE;
assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & assert(arena_mapbits_large_size_get(chunk,
CHUNK_MAP_LARGE) != 0); pageind+head_npages+tail_npages-1) == 0);
assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & assert(arena_mapbits_dirty_get(chunk,
CHUNK_MAP_ALLOCATED) != 0); pageind+head_npages+tail_npages-1) == flag_dirty);
chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits = }
flag_dirty | arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize,
(chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & flag_dirty | arena_mapbits_unzeroed_get(chunk,
CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; pageind+head_npages));
chunk->map[pageind+head_npages-map_bias].bits = (oldsize - newsize) |
flag_dirty | (chunk->map[pageind+head_npages-map_bias].bits &
CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED;
arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize),
dirty); dirty);
@ -1056,12 +1035,13 @@ arena_bin_runs_first(arena_bin_t *bin)
if (mapelm != NULL) { if (mapelm != NULL) {
arena_chunk_t *chunk; arena_chunk_t *chunk;
size_t pageind; size_t pageind;
arena_run_t *run;
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm);
pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) /
sizeof(arena_chunk_map_t))) + map_bias; sizeof(arena_chunk_map_t))) + map_bias;
arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind -
(uintptr_t)((pageind - (mapelm->bits >> LG_PAGE)) << arena_mapbits_small_runind_get(chunk, pageind)) <<
LG_PAGE)); LG_PAGE));
return (run); return (run);
} }
@ -1074,7 +1054,7 @@ arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run)
{ {
arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); arena_chunk_t *chunk = CHUNK_ADDR2BASE(run);
size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind);
assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); assert(arena_run_tree_search(&bin->runs, mapelm) == NULL);
@ -1086,7 +1066,7 @@ arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run)
{ {
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind);
assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); assert(arena_run_tree_search(&bin->runs, mapelm) != NULL);
@ -1125,12 +1105,14 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin)
malloc_mutex_unlock(&bin->lock); malloc_mutex_unlock(&bin->lock);
/******************************/ /******************************/
malloc_mutex_lock(&arena->lock); malloc_mutex_lock(&arena->lock);
run = arena_run_alloc(arena, bin_info->run_size, false, false); run = arena_run_alloc(arena, bin_info->run_size, false, binind, false);
if (run != NULL) { if (run != NULL) {
bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run +
(uintptr_t)bin_info->bitmap_offset); (uintptr_t)bin_info->bitmap_offset);
/* Initialize run internals. */ /* Initialize run internals. */
VALGRIND_MAKE_MEM_UNDEFINED(run, bin_info->reg0_offset -
bin_info->redzone_size);
run->bin = bin; run->bin = bin;
run->nextind = 0; run->nextind = 0;
run->nfree = bin_info->nregs; run->nfree = bin_info->nregs;
@ -1381,7 +1363,7 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero)
/* Large allocation. */ /* Large allocation. */
size = PAGE_CEILING(size); size = PAGE_CEILING(size);
malloc_mutex_lock(&arena->lock); malloc_mutex_lock(&arena->lock);
ret = (void *)arena_run_alloc(arena, size, true, zero); ret = (void *)arena_run_alloc(arena, size, true, BININD_INVALID, zero);
if (ret == NULL) { if (ret == NULL) {
malloc_mutex_unlock(&arena->lock); malloc_mutex_unlock(&arena->lock);
return (NULL); return (NULL);
@ -1425,7 +1407,7 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero)
alloc_size = size + alignment - PAGE; alloc_size = size + alignment - PAGE;
malloc_mutex_lock(&arena->lock); malloc_mutex_lock(&arena->lock);
run = arena_run_alloc(arena, alloc_size, true, zero); run = arena_run_alloc(arena, alloc_size, true, BININD_INVALID, zero);
if (run == NULL) { if (run == NULL) {
malloc_mutex_unlock(&arena->lock); malloc_mutex_unlock(&arena->lock);
return (NULL); return (NULL);
@ -1482,8 +1464,7 @@ arena_prof_promoted(const void *ptr, size_t size)
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
binind = SMALL_SIZE2BIN(size); binind = SMALL_SIZE2BIN(size);
assert(binind < NBINS); assert(binind < NBINS);
chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits & arena_mapbits_large_binind_set(chunk, pageind, binind);
~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT);
assert(isalloc(ptr, false) == PAGE); assert(isalloc(ptr, false) == PAGE);
assert(isalloc(ptr, true) == size); assert(isalloc(ptr, true) == size);
@ -1521,8 +1502,9 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
size_t npages, run_ind, past; size_t npages, run_ind, past;
assert(run != bin->runcur); assert(run != bin->runcur);
assert(arena_run_tree_search(&bin->runs, &chunk->map[ assert(arena_run_tree_search(&bin->runs,
(((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)-map_bias]) == NULL); arena_mapp_get(chunk, ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE))
== NULL);
binind = arena_bin_index(chunk->arena, run->bin); binind = arena_bin_index(chunk->arena, run->bin);
bin_info = &arena_bin_info[binind]; bin_info = &arena_bin_info[binind];
@ -1542,18 +1524,16 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
* trim the clean pages before deallocating the dirty portion of the * trim the clean pages before deallocating the dirty portion of the
* run. * run.
*/ */
if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == 0 && past if (arena_mapbits_dirty_get(chunk, run_ind) == 0 && past - run_ind <
- run_ind < npages) { npages) {
/* /*
* Trim clean pages. Convert to large run beforehand. Set the * Trim clean pages. Convert to large run beforehand. Set the
* last map element first, in case this is a one-page run. * last map element first, in case this is a one-page run.
*/ */
chunk->map[run_ind+npages-1-map_bias].bits = CHUNK_MAP_LARGE | arena_mapbits_large_set(chunk, run_ind+npages-1, 0,
(chunk->map[run_ind+npages-1-map_bias].bits & arena_mapbits_unzeroed_get(chunk, run_ind+npages-1));
CHUNK_MAP_FLAGS_MASK); arena_mapbits_large_set(chunk, run_ind, bin_info->run_size,
chunk->map[run_ind-map_bias].bits = bin_info->run_size | arena_mapbits_unzeroed_get(chunk, run_ind));
CHUNK_MAP_LARGE | (chunk->map[run_ind-map_bias].bits &
CHUNK_MAP_FLAGS_MASK);
arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE), arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE),
((past - run_ind) << LG_PAGE), false); ((past - run_ind) << LG_PAGE), false);
/* npages = past - run_ind; */ /* npages = past - run_ind; */
@ -1588,20 +1568,21 @@ arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
} }
void void
arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr,
arena_chunk_map_t *mapelm) arena_chunk_map_t *mapelm)
{ {
size_t pageind; size_t pageind;
arena_run_t *run; arena_run_t *run;
arena_bin_t *bin; arena_bin_t *bin;
size_t size; arena_bin_info_t *bin_info;
size_t size, binind;
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind -
(mapelm->bits >> LG_PAGE)) << LG_PAGE)); arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE));
bin = run->bin; bin = run->bin;
size_t binind = arena_bin_index(arena, bin); binind = arena_ptr_small_binind_get(ptr, mapelm->bits);
arena_bin_info_t *bin_info = &arena_bin_info[binind]; bin_info = &arena_bin_info[binind];
if (config_fill || config_stats) if (config_fill || config_stats)
size = bin_info->reg_size; size = bin_info->reg_size;
@ -1621,6 +1602,35 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
} }
} }
void
arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
size_t pageind, arena_chunk_map_t *mapelm)
{
arena_run_t *run;
arena_bin_t *bin;
run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind -
arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE));
bin = run->bin;
malloc_mutex_lock(&bin->lock);
arena_dalloc_bin_locked(arena, chunk, ptr, mapelm);
malloc_mutex_unlock(&bin->lock);
}
void
arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr,
size_t pageind)
{
arena_chunk_map_t *mapelm;
if (config_debug) {
/* arena_ptr_small_binind_get() does extra sanity checking. */
assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk,
pageind)) != BININD_INVALID);
}
mapelm = arena_mapp_get(chunk, pageind);
arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm);
}
void void
arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
arena_stats_t *astats, malloc_bin_stats_t *bstats, arena_stats_t *astats, malloc_bin_stats_t *bstats,
@ -1669,12 +1679,12 @@ arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
} }
void void
arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr)
{ {
if (config_fill || config_stats) { if (config_fill || config_stats) {
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
size_t size = chunk->map[pageind-map_bias].bits & ~PAGE_MASK; size_t size = arena_mapbits_large_size_get(chunk, pageind);
if (config_fill && config_stats && opt_junk) if (config_fill && config_stats && opt_junk)
memset(ptr, 0x5a, size); memset(ptr, 0x5a, size);
@ -1689,6 +1699,15 @@ arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
arena_run_dalloc(arena, (arena_run_t *)ptr, true); arena_run_dalloc(arena, (arena_run_t *)ptr, true);
} }
void
arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr)
{
malloc_mutex_lock(&arena->lock);
arena_dalloc_large_locked(arena, chunk, ptr);
malloc_mutex_unlock(&arena->lock);
}
static void static void
arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr,
size_t oldsize, size_t size) size_t oldsize, size_t size)
@ -1727,16 +1746,15 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
size_t npages = oldsize >> LG_PAGE; size_t npages = oldsize >> LG_PAGE;
size_t followsize; size_t followsize;
assert(oldsize == (chunk->map[pageind-map_bias].bits & ~PAGE_MASK)); assert(oldsize == arena_mapbits_large_size_get(chunk, pageind));
/* Try to extend the run. */ /* Try to extend the run. */
assert(size + extra > oldsize); assert(size + extra > oldsize);
malloc_mutex_lock(&arena->lock); malloc_mutex_lock(&arena->lock);
if (pageind + npages < chunk_npages && if (pageind + npages < chunk_npages &&
(chunk->map[pageind+npages-map_bias].bits arena_mapbits_allocated_get(chunk, pageind+npages) == 0 &&
& CHUNK_MAP_ALLOCATED) == 0 && (followsize = (followsize = arena_mapbits_unallocated_size_get(chunk,
chunk->map[pageind+npages-map_bias].bits & ~PAGE_MASK) >= size - pageind+npages)) >= size - oldsize) {
oldsize) {
/* /*
* The next run is available and sufficiently large. Split the * The next run is available and sufficiently large. Split the
* following run, then merge the first part with the existing * following run, then merge the first part with the existing
@ -1746,7 +1764,8 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
size_t splitsize = (oldsize + followsize <= size + extra) size_t splitsize = (oldsize + followsize <= size + extra)
? followsize : size + extra - oldsize; ? followsize : size + extra - oldsize;
arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk +
((pageind+npages) << LG_PAGE)), splitsize, true, zero); ((pageind+npages) << LG_PAGE)), splitsize, true,
BININD_INVALID, zero);
size = oldsize + splitsize; size = oldsize + splitsize;
npages = size >> LG_PAGE; npages = size >> LG_PAGE;
@ -1759,29 +1778,22 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr,
* arena_run_dalloc() with the dirty argument set to false * arena_run_dalloc() with the dirty argument set to false
* (which is when dirty flag consistency would really matter). * (which is when dirty flag consistency would really matter).
*/ */
flag_dirty = (chunk->map[pageind-map_bias].bits & flag_dirty = arena_mapbits_dirty_get(chunk, pageind) |
CHUNK_MAP_DIRTY) | arena_mapbits_dirty_get(chunk, pageind+npages-1);
(chunk->map[pageind+npages-1-map_bias].bits & arena_mapbits_large_set(chunk, pageind, size, flag_dirty);
CHUNK_MAP_DIRTY); arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty);
chunk->map[pageind-map_bias].bits = size | flag_dirty
| CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED;
chunk->map[pageind+npages-1-map_bias].bits = flag_dirty |
CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED;
if (config_stats) { if (config_stats) {
arena->stats.ndalloc_large++; arena->stats.ndalloc_large++;
arena->stats.allocated_large -= oldsize; arena->stats.allocated_large -= oldsize;
arena->stats.lstats[(oldsize >> LG_PAGE) arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++;
- 1].ndalloc++; arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--;
arena->stats.lstats[(oldsize >> LG_PAGE)
- 1].curruns--;
arena->stats.nmalloc_large++; arena->stats.nmalloc_large++;
arena->stats.nrequests_large++; arena->stats.nrequests_large++;
arena->stats.allocated_large += size; arena->stats.allocated_large += size;
arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++;
arena->stats.lstats[(size >> LG_PAGE) arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++;
- 1].nrequests++;
arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++;
} }
malloc_mutex_unlock(&arena->lock); malloc_mutex_unlock(&arena->lock);
@ -1924,6 +1936,7 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
* expectation that the extra bytes will be reliably preserved. * expectation that the extra bytes will be reliably preserved.
*/ */
copysize = (size < oldsize) ? size : oldsize; copysize = (size < oldsize) ? size : oldsize;
VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize);
memcpy(ret, ptr, copysize); memcpy(ret, ptr, copysize);
iqalloc(ptr); iqalloc(ptr);
return (ret); return (ret);

97
contrib/jemalloc/src/chunk.c
View File

@ -30,19 +30,30 @@ size_t arena_maxclass; /* Max size class for arenas. */
/******************************************************************************/ /******************************************************************************/
/* Function prototypes for non-inline static functions. */ /* Function prototypes for non-inline static functions. */
static void *chunk_recycle(size_t size, size_t alignment, bool *zero); static void *chunk_recycle(size_t size, size_t alignment, bool base,
bool *zero);
static void chunk_record(void *chunk, size_t size); static void chunk_record(void *chunk, size_t size);
/******************************************************************************/ /******************************************************************************/
static void * static void *
chunk_recycle(size_t size, size_t alignment, bool *zero) chunk_recycle(size_t size, size_t alignment, bool base, bool *zero)
{ {
void *ret; void *ret;
extent_node_t *node; extent_node_t *node;
extent_node_t key; extent_node_t key;
size_t alloc_size, leadsize, trailsize; size_t alloc_size, leadsize, trailsize;
if (base) {
/*
* This function may need to call base_node_{,de}alloc(), but
* the current chunk allocation request is on behalf of the
* base allocator. Avoid deadlock (and if that weren't an
* issue, potential for infinite recursion) by returning NULL.
*/
return (NULL);
}
alloc_size = size + alignment - chunksize; alloc_size = size + alignment - chunksize;
/* Beware size_t wrap-around. */ /* Beware size_t wrap-around. */
if (alloc_size < size) if (alloc_size < size)
@ -57,8 +68,8 @@ chunk_recycle(size_t size, size_t alignment, bool *zero)
} }
leadsize = ALIGNMENT_CEILING((uintptr_t)node->addr, alignment) - leadsize = ALIGNMENT_CEILING((uintptr_t)node->addr, alignment) -
(uintptr_t)node->addr; (uintptr_t)node->addr;
assert(alloc_size >= leadsize + size); assert(node->size >= leadsize + size);
trailsize = alloc_size - leadsize - size; trailsize = node->size - leadsize - size;
ret = (void *)((uintptr_t)node->addr + leadsize); ret = (void *)((uintptr_t)node->addr + leadsize);
/* Remove node from the tree. */ /* Remove node from the tree. */
extent_tree_szad_remove(&chunks_szad, node); extent_tree_szad_remove(&chunks_szad, node);
@ -123,9 +134,10 @@ chunk_alloc(size_t size, size_t alignment, bool base, bool *zero)
assert(size != 0); assert(size != 0);
assert((size & chunksize_mask) == 0); assert((size & chunksize_mask) == 0);
assert(alignment != 0);
assert((alignment & chunksize_mask) == 0); assert((alignment & chunksize_mask) == 0);
ret = chunk_recycle(size, alignment, zero); ret = chunk_recycle(size, alignment, base, zero);
if (ret != NULL) if (ret != NULL)
goto label_return; goto label_return;
@ -168,6 +180,7 @@ chunk_alloc(size_t size, size_t alignment, bool base, bool *zero)
size_t i; size_t i;
size_t *p = (size_t *)(uintptr_t)ret; size_t *p = (size_t *)(uintptr_t)ret;
VALGRIND_MAKE_MEM_DEFINED(ret, size);
for (i = 0; i < size / sizeof(size_t); i++) for (i = 0; i < size / sizeof(size_t); i++)
assert(p[i] == 0); assert(p[i] == 0);
} }
@ -182,50 +195,48 @@ chunk_record(void *chunk, size_t size)
pages_purge(chunk, size); pages_purge(chunk, size);
xnode = NULL; /*
* Allocate a node before acquiring chunks_mtx even though it might not
* be needed, because base_node_alloc() may cause a new base chunk to
* be allocated, which could cause deadlock if chunks_mtx were already
* held.
*/
xnode = base_node_alloc();
malloc_mutex_lock(&chunks_mtx); malloc_mutex_lock(&chunks_mtx);
while (true) { key.addr = (void *)((uintptr_t)chunk + size);
key.addr = (void *)((uintptr_t)chunk + size); node = extent_tree_ad_nsearch(&chunks_ad, &key);
node = extent_tree_ad_nsearch(&chunks_ad, &key); /* Try to coalesce forward. */
/* Try to coalesce forward. */ if (node != NULL && node->addr == key.addr) {
if (node != NULL && node->addr == key.addr) { /*
* Coalesce chunk with the following address range. This does
* not change the position within chunks_ad, so only
* remove/insert from/into chunks_szad.
*/
extent_tree_szad_remove(&chunks_szad, node);
node->addr = chunk;
node->size += size;
extent_tree_szad_insert(&chunks_szad, node);
if (xnode != NULL)
base_node_dealloc(xnode);
} else {
/* Coalescing forward failed, so insert a new node. */
if (xnode == NULL) {
/* /*
* Coalesce chunk with the following address range. * base_node_alloc() failed, which is an exceedingly
* This does not change the position within chunks_ad, * unlikely failure. Leak chunk; its pages have
* so only remove/insert from/into chunks_szad. * already been purged, so this is only a virtual
*/ * memory leak.
extent_tree_szad_remove(&chunks_szad, node);
node->addr = chunk;
node->size += size;
extent_tree_szad_insert(&chunks_szad, node);
break;
} else if (xnode == NULL) {
/*
* It is possible that base_node_alloc() will cause a
* new base chunk to be allocated, so take care not to
* deadlock on chunks_mtx, and recover if another thread
* deallocates an adjacent chunk while this one is busy
* allocating xnode.
*/ */
malloc_mutex_unlock(&chunks_mtx); malloc_mutex_unlock(&chunks_mtx);
xnode = base_node_alloc(); return;
if (xnode == NULL)
return;
malloc_mutex_lock(&chunks_mtx);
} else {
/* Coalescing forward failed, so insert a new node. */
node = xnode;
xnode = NULL;
node->addr = chunk;
node->size = size;
extent_tree_ad_insert(&chunks_ad, node);
extent_tree_szad_insert(&chunks_szad, node);
break;
} }
node = xnode;
node->addr = chunk;
node->size = size;
extent_tree_ad_insert(&chunks_ad, node);
extent_tree_szad_insert(&chunks_szad, node);
} }
/* Discard xnode if it ended up unused due to a race. */
if (xnode != NULL)
base_node_dealloc(xnode);
/* Try to coalesce backward. */ /* Try to coalesce backward. */
prev = extent_tree_ad_prev(&chunks_ad, node); prev = extent_tree_ad_prev(&chunks_ad, node);

149
contrib/jemalloc/src/chunk_mmap.c
View File

@ -7,7 +7,7 @@
static void *pages_map(void *addr, size_t size); static void *pages_map(void *addr, size_t size);
static void pages_unmap(void *addr, size_t size); static void pages_unmap(void *addr, size_t size);
static void *chunk_alloc_mmap_slow(size_t size, size_t alignment, static void *chunk_alloc_mmap_slow(size_t size, size_t alignment,
bool unaligned, bool *zero); bool *zero);
/******************************************************************************/ /******************************************************************************/
@ -16,6 +16,16 @@ pages_map(void *addr, size_t size)
{ {
void *ret; void *ret;
assert(size != 0);
#ifdef _WIN32
/*
* If VirtualAlloc can't allocate at the given address when one is
* given, it fails and returns NULL.
*/
ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
PAGE_READWRITE);
#else
/* /*
* We don't use MAP_FIXED here, because it can cause the *replacement* * We don't use MAP_FIXED here, because it can cause the *replacement*
* of existing mappings, and we only want to create new mappings. * of existing mappings, and we only want to create new mappings.
@ -33,7 +43,7 @@ pages_map(void *addr, size_t size)
if (munmap(ret, size) == -1) { if (munmap(ret, size) == -1) {
char buf[BUFERROR_BUF]; char buf[BUFERROR_BUF];
buferror(errno, buf, sizeof(buf)); buferror(buf, sizeof(buf));
malloc_printf("<jemalloc: Error in munmap(): %s\n", malloc_printf("<jemalloc: Error in munmap(): %s\n",
buf); buf);
if (opt_abort) if (opt_abort)
@ -41,7 +51,7 @@ pages_map(void *addr, size_t size)
} }
ret = NULL; ret = NULL;
} }
#endif
assert(ret == NULL || (addr == NULL && ret != addr) assert(ret == NULL || (addr == NULL && ret != addr)
|| (addr != NULL && ret == addr)); || (addr != NULL && ret == addr));
return (ret); return (ret);
@ -51,55 +61,94 @@ static void
pages_unmap(void *addr, size_t size) pages_unmap(void *addr, size_t size)
{ {
if (munmap(addr, size) == -1) { #ifdef _WIN32
if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
#else
if (munmap(addr, size) == -1)
#endif
{
char buf[BUFERROR_BUF]; char buf[BUFERROR_BUF];
buferror(errno, buf, sizeof(buf)); buferror(buf, sizeof(buf));
malloc_printf("<jemalloc>: Error in munmap(): %s\n", buf); malloc_printf("<jemalloc>: Error in "
#ifdef _WIN32
"VirtualFree"
#else
"munmap"
#endif
"(): %s\n", buf);
if (opt_abort) if (opt_abort)
abort(); abort();
} }
} }
static void *
pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size)
{
void *ret = (void *)((uintptr_t)addr + leadsize);
assert(alloc_size >= leadsize + size);
#ifdef _WIN32
{
void *new_addr;
pages_unmap(addr, alloc_size);
new_addr = pages_map(ret, size);
if (new_addr == ret)
return (ret);
if (new_addr)
pages_unmap(new_addr, size);
return (NULL);
}
#else
{
size_t trailsize = alloc_size - leadsize - size;
if (leadsize != 0)
pages_unmap(addr, leadsize);
if (trailsize != 0)
pages_unmap((void *)((uintptr_t)ret + size), trailsize);
return (ret);
}
#endif
}
void void
pages_purge(void *addr, size_t length) pages_purge(void *addr, size_t length)
{ {
#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED #ifdef _WIN32
# define JEMALLOC_MADV_PURGE MADV_DONTNEED VirtualAlloc(addr, length, MEM_RESET, PAGE_READWRITE);
#elif defined(JEMALLOC_PURGE_MADVISE_FREE)
# define JEMALLOC_MADV_PURGE MADV_FREE
#else #else
# error "No method defined for purging unused dirty pages." # ifdef JEMALLOC_PURGE_MADVISE_DONTNEED
#endif # define JEMALLOC_MADV_PURGE MADV_DONTNEED
# elif defined(JEMALLOC_PURGE_MADVISE_FREE)
# define JEMALLOC_MADV_PURGE MADV_FREE
# else
# error "No method defined for purging unused dirty pages."
# endif
madvise(addr, length, JEMALLOC_MADV_PURGE); madvise(addr, length, JEMALLOC_MADV_PURGE);
#endif
} }
static void * static void *
chunk_alloc_mmap_slow(size_t size, size_t alignment, bool unaligned, bool *zero) chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero)
{ {
void *ret, *pages; void *ret, *pages;
size_t alloc_size, leadsize, trailsize; size_t alloc_size, leadsize;
alloc_size = size + alignment - PAGE; alloc_size = size + alignment - PAGE;
/* Beware size_t wrap-around. */ /* Beware size_t wrap-around. */
if (alloc_size < size) if (alloc_size < size)
return (NULL); return (NULL);
pages = pages_map(NULL, alloc_size); do {
if (pages == NULL) pages = pages_map(NULL, alloc_size);
return (NULL); if (pages == NULL)
leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) - return (NULL);
(uintptr_t)pages; leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) -
assert(alloc_size >= leadsize + size); (uintptr_t)pages;
trailsize = alloc_size - leadsize - size; ret = pages_trim(pages, alloc_size, leadsize, size);
ret = (void *)((uintptr_t)pages + leadsize); } while (ret == NULL);
if (leadsize != 0) {
/* Note that mmap() returned an unaligned mapping. */
unaligned = true;
pages_unmap(pages, leadsize);
}
if (trailsize != 0)
pages_unmap((void *)((uintptr_t)ret + size), trailsize);
assert(ret != NULL); assert(ret != NULL);
*zero = true; *zero = true;
@ -117,48 +166,24 @@ chunk_alloc_mmap(size_t size, size_t alignment, bool *zero)
* NetBSD has), but in the absence of such a feature, we have to work * NetBSD has), but in the absence of such a feature, we have to work
* hard to efficiently create aligned mappings. The reliable, but * hard to efficiently create aligned mappings. The reliable, but
* slow method is to create a mapping that is over-sized, then trim the * slow method is to create a mapping that is over-sized, then trim the
* excess. However, that always results in at least one call to * excess. However, that always results in one or two calls to
* pages_unmap(). * pages_unmap().
* *
* A more optimistic approach is to try mapping precisely the right * Optimistically try mapping precisely the right amount before falling
* amount, then try to append another mapping if alignment is off. In * back to the slow method, with the expectation that the optimistic
* practice, this works out well as long as the application is not * approach works most of the time.
* interleaving mappings via direct mmap() calls. If we do run into a
* situation where there is an interleaved mapping and we are unable to
* extend an unaligned mapping, our best option is to switch to the
* slow method until mmap() returns another aligned mapping. This will
* tend to leave a gap in the memory map that is too small to cause
* later problems for the optimistic method.
*
* Another possible confounding factor is address space layout
* randomization (ASLR), which causes mmap(2) to disregard the
* requested address. As such, repeatedly trying to extend unaligned
* mappings could result in an infinite loop, so if extension fails,
* immediately fall back to the reliable method of over-allocation
* followed by trimming.
*/ */
assert(alignment != 0);
assert((alignment & chunksize_mask) == 0);
ret = pages_map(NULL, size); ret = pages_map(NULL, size);
if (ret == NULL) if (ret == NULL)
return (NULL); return (NULL);
offset = ALIGNMENT_ADDR2OFFSET(ret, alignment); offset = ALIGNMENT_ADDR2OFFSET(ret, alignment);
if (offset != 0) { if (offset != 0) {
/* Try to extend chunk boundary. */ pages_unmap(ret, size);
if (pages_map((void *)((uintptr_t)ret + size), chunksize - return (chunk_alloc_mmap_slow(size, alignment, zero));
offset) == NULL) {
/*
* Extension failed. Clean up, then fall back to the
* reliable-but-expensive method.
*/
pages_unmap(ret, size);
return (chunk_alloc_mmap_slow(size, alignment, true,
zero));
} else {
/* Clean up unneeded leading space. */
pages_unmap(ret, chunksize - offset);
ret = (void *)((uintptr_t)ret + (chunksize - offset));
}
} }
assert(ret != NULL); assert(ret != NULL);

252
contrib/jemalloc/src/ctl.c
View File

@ -14,6 +14,32 @@ static bool ctl_initialized;
static uint64_t ctl_epoch; static uint64_t ctl_epoch;
static ctl_stats_t ctl_stats; static ctl_stats_t ctl_stats;
/******************************************************************************/
/* Helpers for named and indexed nodes. */
static inline const ctl_named_node_t *
ctl_named_node(const ctl_node_t *node)
{
return ((node->named) ? (const ctl_named_node_t *)node : NULL);
}
static inline const ctl_named_node_t *
ctl_named_children(const ctl_named_node_t *node, int index)
{
const ctl_named_node_t *children = ctl_named_node(node->children);
return (children ? &children[index] : NULL);
}
static inline const ctl_indexed_node_t *
ctl_indexed_node(const ctl_node_t *node)
{
return ((node->named == false) ? (const ctl_indexed_node_t *)node :
NULL);
}
/******************************************************************************/ /******************************************************************************/
/* Function prototypes for non-inline static functions. */ /* Function prototypes for non-inline static functions. */
@ -22,7 +48,7 @@ static int n##_ctl(const size_t *mib, size_t miblen, void *oldp, \
size_t *oldlenp, void *newp, size_t newlen); size_t *oldlenp, void *newp, size_t newlen);
#define INDEX_PROTO(n) \ #define INDEX_PROTO(n) \
const ctl_node_t *n##_index(const size_t *mib, size_t miblen, \ const ctl_named_node_t *n##_index(const size_t *mib, size_t miblen, \
size_t i); size_t i);
static bool ctl_arena_init(ctl_arena_stats_t *astats); static bool ctl_arena_init(ctl_arena_stats_t *astats);
@ -50,6 +76,7 @@ CTL_PROTO(config_debug)
CTL_PROTO(config_dss) CTL_PROTO(config_dss)
CTL_PROTO(config_fill) CTL_PROTO(config_fill)
CTL_PROTO(config_lazy_lock) CTL_PROTO(config_lazy_lock)
CTL_PROTO(config_mremap)
CTL_PROTO(config_munmap) CTL_PROTO(config_munmap)
CTL_PROTO(config_prof) CTL_PROTO(config_prof)
CTL_PROTO(config_prof_libgcc) CTL_PROTO(config_prof_libgcc)
@ -149,35 +176,39 @@ CTL_PROTO(stats_mapped)
/* Maximum tree depth. */ /* Maximum tree depth. */
#define CTL_MAX_DEPTH 6 #define CTL_MAX_DEPTH 6
#define NAME(n) true, {.named = {n #define NAME(n) {true}, n
#define CHILD(c) sizeof(c##_node) / sizeof(ctl_node_t), c##_node}}, NULL #define CHILD(t, c) \
#define CTL(c) 0, NULL}}, c##_ctl sizeof(c##_node) / sizeof(ctl_##t##_node_t), \
(ctl_node_t *)c##_node, \
NULL
#define CTL(c) 0, NULL, c##_ctl
/* /*
* Only handles internal indexed nodes, since there are currently no external * Only handles internal indexed nodes, since there are currently no external
* ones. * ones.
*/ */
#define INDEX(i) false, {.indexed = {i##_index}}, NULL #define INDEX(i) {false}, i##_index
static const ctl_node_t tcache_node[] = { static const ctl_named_node_t tcache_node[] = {
{NAME("enabled"), CTL(thread_tcache_enabled)}, {NAME("enabled"), CTL(thread_tcache_enabled)},
{NAME("flush"), CTL(thread_tcache_flush)} {NAME("flush"), CTL(thread_tcache_flush)}
}; };
static const ctl_node_t thread_node[] = { static const ctl_named_node_t thread_node[] = {
{NAME("arena"), CTL(thread_arena)}, {NAME("arena"), CTL(thread_arena)},
{NAME("allocated"), CTL(thread_allocated)}, {NAME("allocated"), CTL(thread_allocated)},
{NAME("allocatedp"), CTL(thread_allocatedp)}, {NAME("allocatedp"), CTL(thread_allocatedp)},
{NAME("deallocated"), CTL(thread_deallocated)}, {NAME("deallocated"), CTL(thread_deallocated)},
{NAME("deallocatedp"), CTL(thread_deallocatedp)}, {NAME("deallocatedp"), CTL(thread_deallocatedp)},
{NAME("tcache"), CHILD(tcache)} {NAME("tcache"), CHILD(named, tcache)}
}; };
static const ctl_node_t config_node[] = { static const ctl_named_node_t config_node[] = {
{NAME("debug"), CTL(config_debug)}, {NAME("debug"), CTL(config_debug)},
{NAME("dss"), CTL(config_dss)}, {NAME("dss"), CTL(config_dss)},
{NAME("fill"), CTL(config_fill)}, {NAME("fill"), CTL(config_fill)},
{NAME("lazy_lock"), CTL(config_lazy_lock)}, {NAME("lazy_lock"), CTL(config_lazy_lock)},
{NAME("mremap"), CTL(config_mremap)},
{NAME("munmap"), CTL(config_munmap)}, {NAME("munmap"), CTL(config_munmap)},
{NAME("prof"), CTL(config_prof)}, {NAME("prof"), CTL(config_prof)},
{NAME("prof_libgcc"), CTL(config_prof_libgcc)}, {NAME("prof_libgcc"), CTL(config_prof_libgcc)},
@ -190,7 +221,7 @@ static const ctl_node_t config_node[] = {
{NAME("xmalloc"), CTL(config_xmalloc)} {NAME("xmalloc"), CTL(config_xmalloc)}
}; };
static const ctl_node_t opt_node[] = { static const ctl_named_node_t opt_node[] = {
{NAME("abort"), CTL(opt_abort)}, {NAME("abort"), CTL(opt_abort)},
{NAME("lg_chunk"), CTL(opt_lg_chunk)}, {NAME("lg_chunk"), CTL(opt_lg_chunk)},
{NAME("narenas"), CTL(opt_narenas)}, {NAME("narenas"), CTL(opt_narenas)},
@ -216,31 +247,31 @@ static const ctl_node_t opt_node[] = {
{NAME("prof_accum"), CTL(opt_prof_accum)} {NAME("prof_accum"), CTL(opt_prof_accum)}
}; };
static const ctl_node_t arenas_bin_i_node[] = { static const ctl_named_node_t arenas_bin_i_node[] = {
{NAME("size"), CTL(arenas_bin_i_size)}, {NAME("size"), CTL(arenas_bin_i_size)},
{NAME("nregs"), CTL(arenas_bin_i_nregs)}, {NAME("nregs"), CTL(arenas_bin_i_nregs)},
{NAME("run_size"), CTL(arenas_bin_i_run_size)} {NAME("run_size"), CTL(arenas_bin_i_run_size)}
}; };
static const ctl_node_t super_arenas_bin_i_node[] = { static const ctl_named_node_t super_arenas_bin_i_node[] = {
{NAME(""), CHILD(arenas_bin_i)} {NAME(""), CHILD(named, arenas_bin_i)}
}; };
static const ctl_node_t arenas_bin_node[] = { static const ctl_indexed_node_t arenas_bin_node[] = {
{INDEX(arenas_bin_i)} {INDEX(arenas_bin_i)}
}; };
static const ctl_node_t arenas_lrun_i_node[] = { static const ctl_named_node_t arenas_lrun_i_node[] = {
{NAME("size"), CTL(arenas_lrun_i_size)} {NAME("size"), CTL(arenas_lrun_i_size)}
}; };
static const ctl_node_t super_arenas_lrun_i_node[] = { static const ctl_named_node_t super_arenas_lrun_i_node[] = {
{NAME(""), CHILD(arenas_lrun_i)} {NAME(""), CHILD(named, arenas_lrun_i)}
}; };
static const ctl_node_t arenas_lrun_node[] = { static const ctl_indexed_node_t arenas_lrun_node[] = {
{INDEX(arenas_lrun_i)} {INDEX(arenas_lrun_i)}
}; };
static const ctl_node_t arenas_node[] = { static const ctl_named_node_t arenas_node[] = {
{NAME("narenas"), CTL(arenas_narenas)}, {NAME("narenas"), CTL(arenas_narenas)},
{NAME("initialized"), CTL(arenas_initialized)}, {NAME("initialized"), CTL(arenas_initialized)},
{NAME("quantum"), CTL(arenas_quantum)}, {NAME("quantum"), CTL(arenas_quantum)},
@ -248,45 +279,45 @@ static const ctl_node_t arenas_node[] = {
{NAME("tcache_max"), CTL(arenas_tcache_max)}, {NAME("tcache_max"), CTL(arenas_tcache_max)},
{NAME("nbins"), CTL(arenas_nbins)}, {NAME("nbins"), CTL(arenas_nbins)},
{NAME("nhbins"), CTL(arenas_nhbins)}, {NAME("nhbins"), CTL(arenas_nhbins)},
{NAME("bin"), CHILD(arenas_bin)}, {NAME("bin"), CHILD(indexed, arenas_bin)},
{NAME("nlruns"), CTL(arenas_nlruns)}, {NAME("nlruns"), CTL(arenas_nlruns)},
{NAME("lrun"), CHILD(arenas_lrun)}, {NAME("lrun"), CHILD(indexed, arenas_lrun)},
{NAME("purge"), CTL(arenas_purge)} {NAME("purge"), CTL(arenas_purge)}
}; };
static const ctl_node_t prof_node[] = { static const ctl_named_node_t prof_node[] = {
{NAME("active"), CTL(prof_active)}, {NAME("active"), CTL(prof_active)},
{NAME("dump"), CTL(prof_dump)}, {NAME("dump"), CTL(prof_dump)},
{NAME("interval"), CTL(prof_interval)} {NAME("interval"), CTL(prof_interval)}
}; };
static const ctl_node_t stats_chunks_node[] = { static const ctl_named_node_t stats_chunks_node[] = {
{NAME("current"), CTL(stats_chunks_current)}, {NAME("current"), CTL(stats_chunks_current)},
{NAME("total"), CTL(stats_chunks_total)}, {NAME("total"), CTL(stats_chunks_total)},
{NAME("high"), CTL(stats_chunks_high)} {NAME("high"), CTL(stats_chunks_high)}
}; };
static const ctl_node_t stats_huge_node[] = { static const ctl_named_node_t stats_huge_node[] = {
{NAME("allocated"), CTL(stats_huge_allocated)}, {NAME("allocated"), CTL(stats_huge_allocated)},
{NAME("nmalloc"), CTL(stats_huge_nmalloc)}, {NAME("nmalloc"), CTL(stats_huge_nmalloc)},
{NAME("ndalloc"), CTL(stats_huge_ndalloc)} {NAME("ndalloc"), CTL(stats_huge_ndalloc)}
}; };
static const ctl_node_t stats_arenas_i_small_node[] = { static const ctl_named_node_t stats_arenas_i_small_node[] = {
{NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, {NAME("allocated"), CTL(stats_arenas_i_small_allocated)},
{NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)},
{NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)},
{NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)}
}; };
static const ctl_node_t stats_arenas_i_large_node[] = { static const ctl_named_node_t stats_arenas_i_large_node[] = {
{NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, {NAME("allocated"), CTL(stats_arenas_i_large_allocated)},
{NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)},
{NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)},
{NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)}
}; };
static const ctl_node_t stats_arenas_i_bins_j_node[] = { static const ctl_named_node_t stats_arenas_i_bins_j_node[] = {
{NAME("allocated"), CTL(stats_arenas_i_bins_j_allocated)}, {NAME("allocated"), CTL(stats_arenas_i_bins_j_allocated)},
{NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)},
{NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)},
@ -297,29 +328,29 @@ static const ctl_node_t stats_arenas_i_bins_j_node[] = {
{NAME("nreruns"), CTL(stats_arenas_i_bins_j_nreruns)}, {NAME("nreruns"), CTL(stats_arenas_i_bins_j_nreruns)},
{NAME("curruns"), CTL(stats_arenas_i_bins_j_curruns)} {NAME("curruns"), CTL(stats_arenas_i_bins_j_curruns)}
}; };
static const ctl_node_t super_stats_arenas_i_bins_j_node[] = { static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = {
{NAME(""), CHILD(stats_arenas_i_bins_j)} {NAME(""), CHILD(named, stats_arenas_i_bins_j)}
}; };
static const ctl_node_t stats_arenas_i_bins_node[] = { static const ctl_indexed_node_t stats_arenas_i_bins_node[] = {
{INDEX(stats_arenas_i_bins_j)} {INDEX(stats_arenas_i_bins_j)}
}; };
static const ctl_node_t stats_arenas_i_lruns_j_node[] = { static const ctl_named_node_t stats_arenas_i_lruns_j_node[] = {
{NAME("nmalloc"), CTL(stats_arenas_i_lruns_j_nmalloc)}, {NAME("nmalloc"), CTL(stats_arenas_i_lruns_j_nmalloc)},
{NAME("ndalloc"), CTL(stats_arenas_i_lruns_j_ndalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_lruns_j_ndalloc)},
{NAME("nrequests"), CTL(stats_arenas_i_lruns_j_nrequests)}, {NAME("nrequests"), CTL(stats_arenas_i_lruns_j_nrequests)},
{NAME("curruns"), CTL(stats_arenas_i_lruns_j_curruns)} {NAME("curruns"), CTL(stats_arenas_i_lruns_j_curruns)}
}; };
static const ctl_node_t super_stats_arenas_i_lruns_j_node[] = { static const ctl_named_node_t super_stats_arenas_i_lruns_j_node[] = {
{NAME(""), CHILD(stats_arenas_i_lruns_j)} {NAME(""), CHILD(named, stats_arenas_i_lruns_j)}
}; };
static const ctl_node_t stats_arenas_i_lruns_node[] = { static const ctl_indexed_node_t stats_arenas_i_lruns_node[] = {
{INDEX(stats_arenas_i_lruns_j)} {INDEX(stats_arenas_i_lruns_j)}
}; };
static const ctl_node_t stats_arenas_i_node[] = { static const ctl_named_node_t stats_arenas_i_node[] = {
{NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, {NAME("nthreads"), CTL(stats_arenas_i_nthreads)},
{NAME("pactive"), CTL(stats_arenas_i_pactive)}, {NAME("pactive"), CTL(stats_arenas_i_pactive)},
{NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, {NAME("pdirty"), CTL(stats_arenas_i_pdirty)},
@ -327,41 +358,41 @@ static const ctl_node_t stats_arenas_i_node[] = {
{NAME("npurge"), CTL(stats_arenas_i_npurge)}, {NAME("npurge"), CTL(stats_arenas_i_npurge)},
{NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)}, {NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)},
{NAME("purged"), CTL(stats_arenas_i_purged)}, {NAME("purged"), CTL(stats_arenas_i_purged)},
{NAME("small"), CHILD(stats_arenas_i_small)}, {NAME("small"), CHILD(named, stats_arenas_i_small)},
{NAME("large"), CHILD(stats_arenas_i_large)}, {NAME("large"), CHILD(named, stats_arenas_i_large)},
{NAME("bins"), CHILD(stats_arenas_i_bins)}, {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
{NAME("lruns"), CHILD(stats_arenas_i_lruns)} {NAME("lruns"), CHILD(indexed, stats_arenas_i_lruns)}
}; };
static const ctl_node_t super_stats_arenas_i_node[] = { static const ctl_named_node_t super_stats_arenas_i_node[] = {
{NAME(""), CHILD(stats_arenas_i)} {NAME(""), CHILD(named, stats_arenas_i)}
}; };
static const ctl_node_t stats_arenas_node[] = { static const ctl_indexed_node_t stats_arenas_node[] = {
{INDEX(stats_arenas_i)} {INDEX(stats_arenas_i)}
}; };
static const ctl_node_t stats_node[] = { static const ctl_named_node_t stats_node[] = {
{NAME("cactive"), CTL(stats_cactive)}, {NAME("cactive"), CTL(stats_cactive)},
{NAME("allocated"), CTL(stats_allocated)}, {NAME("allocated"), CTL(stats_allocated)},
{NAME("active"), CTL(stats_active)}, {NAME("active"), CTL(stats_active)},
{NAME("mapped"), CTL(stats_mapped)}, {NAME("mapped"), CTL(stats_mapped)},
{NAME("chunks"), CHILD(stats_chunks)}, {NAME("chunks"), CHILD(named, stats_chunks)},
{NAME("huge"), CHILD(stats_huge)}, {NAME("huge"), CHILD(named, stats_huge)},
{NAME("arenas"), CHILD(stats_arenas)} {NAME("arenas"), CHILD(indexed, stats_arenas)}
}; };
static const ctl_node_t root_node[] = { static const ctl_named_node_t root_node[] = {
{NAME("version"), CTL(version)}, {NAME("version"), CTL(version)},
{NAME("epoch"), CTL(epoch)}, {NAME("epoch"), CTL(epoch)},
{NAME("thread"), CHILD(thread)}, {NAME("thread"), CHILD(named, thread)},
{NAME("config"), CHILD(config)}, {NAME("config"), CHILD(named, config)},
{NAME("opt"), CHILD(opt)}, {NAME("opt"), CHILD(named, opt)},
{NAME("arenas"), CHILD(arenas)}, {NAME("arenas"), CHILD(named, arenas)},
{NAME("prof"), CHILD(prof)}, {NAME("prof"), CHILD(named, prof)},
{NAME("stats"), CHILD(stats)} {NAME("stats"), CHILD(named, stats)}
}; };
static const ctl_node_t super_root_node[] = { static const ctl_named_node_t super_root_node[] = {
{NAME(""), CHILD(root)} {NAME(""), CHILD(named, root)}
}; };
#undef NAME #undef NAME
@ -491,7 +522,7 @@ static void
ctl_refresh(void) ctl_refresh(void)
{ {
unsigned i; unsigned i;
arena_t *tarenas[narenas]; VARIABLE_ARRAY(arena_t *, tarenas, narenas);
if (config_stats) { if (config_stats) {
malloc_mutex_lock(&chunks_mtx); malloc_mutex_lock(&chunks_mtx);
@ -597,7 +628,7 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp,
int ret; int ret;
const char *elm, *tdot, *dot; const char *elm, *tdot, *dot;
size_t elen, i, j; size_t elen, i, j;
const ctl_node_t *node; const ctl_named_node_t *node;
elm = name; elm = name;
/* Equivalent to strchrnul(). */ /* Equivalent to strchrnul(). */
@ -609,21 +640,21 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp,
} }
node = super_root_node; node = super_root_node;
for (i = 0; i < *depthp; i++) { for (i = 0; i < *depthp; i++) {
assert(node->named); assert(node);
assert(node->u.named.nchildren > 0); assert(node->nchildren > 0);
if (node->u.named.children[0].named) { if (ctl_named_node(node->children) != NULL) {
const ctl_node_t *pnode = node; const ctl_named_node_t *pnode = node;
/* Children are named. */ /* Children are named. */
for (j = 0; j < node->u.named.nchildren; j++) { for (j = 0; j < node->nchildren; j++) {
const ctl_node_t *child = const ctl_named_node_t *child =
&node->u.named.children[j]; ctl_named_children(node, j);
if (strlen(child->u.named.name) == elen if (strlen(child->name) == elen &&
&& strncmp(elm, child->u.named.name, strncmp(elm, child->name, elen) == 0) {
elen) == 0) {
node = child; node = child;
if (nodesp != NULL) if (nodesp != NULL)
nodesp[i] = node; nodesp[i] =
(const ctl_node_t *)node;
mibp[i] = j; mibp[i] = j;
break; break;
} }
@ -634,7 +665,7 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp,
} }
} else { } else {
uintmax_t index; uintmax_t index;
const ctl_node_t *inode; const ctl_indexed_node_t *inode;
/* Children are indexed. */ /* Children are indexed. */
index = malloc_strtoumax(elm, NULL, 10); index = malloc_strtoumax(elm, NULL, 10);
@ -643,16 +674,15 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp,
goto label_return; goto label_return;
} }
inode = &node->u.named.children[0]; inode = ctl_indexed_node(node->children);
node = inode->u.indexed.index(mibp, *depthp, node = inode->index(mibp, *depthp, (size_t)index);
(size_t)index);
if (node == NULL) { if (node == NULL) {
ret = ENOENT; ret = ENOENT;
goto label_return; goto label_return;
} }
if (nodesp != NULL) if (nodesp != NULL)
nodesp[i] = node; nodesp[i] = (const ctl_node_t *)node;
mibp[i] = (size_t)index; mibp[i] = (size_t)index;
} }
@ -696,6 +726,7 @@ ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp,
size_t depth; size_t depth;
ctl_node_t const *nodes[CTL_MAX_DEPTH]; ctl_node_t const *nodes[CTL_MAX_DEPTH];
size_t mib[CTL_MAX_DEPTH]; size_t mib[CTL_MAX_DEPTH];
const ctl_named_node_t *node;
if (ctl_initialized == false && ctl_init()) { if (ctl_initialized == false && ctl_init()) {
ret = EAGAIN; ret = EAGAIN;
@ -707,13 +738,14 @@ ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp,
if (ret != 0) if (ret != 0)
goto label_return; goto label_return;
if (nodes[depth-1]->ctl == NULL) { node = ctl_named_node(nodes[depth-1]);
if (node != NULL && node->ctl)
ret = node->ctl(mib, depth, oldp, oldlenp, newp, newlen);
else {
/* The name refers to a partial path through the ctl tree. */ /* The name refers to a partial path through the ctl tree. */
ret = ENOENT; ret = ENOENT;
goto label_return;
} }
ret = nodes[depth-1]->ctl(mib, depth, oldp, oldlenp, newp, newlen);
label_return: label_return:
return(ret); return(ret);
} }
@ -738,7 +770,7 @@ ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
void *newp, size_t newlen) void *newp, size_t newlen)
{ {
int ret; int ret;
const ctl_node_t *node; const ctl_named_node_t *node;
size_t i; size_t i;
if (ctl_initialized == false && ctl_init()) { if (ctl_initialized == false && ctl_init()) {
@ -749,19 +781,21 @@ ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
/* Iterate down the tree. */ /* Iterate down the tree. */
node = super_root_node; node = super_root_node;
for (i = 0; i < miblen; i++) { for (i = 0; i < miblen; i++) {
if (node->u.named.children[0].named) { assert(node);
assert(node->nchildren > 0);
if (ctl_named_node(node->children) != NULL) {
/* Children are named. */ /* Children are named. */
if (node->u.named.nchildren <= mib[i]) { if (node->nchildren <= mib[i]) {
ret = ENOENT; ret = ENOENT;
goto label_return; goto label_return;
} }
node = &node->u.named.children[mib[i]]; node = ctl_named_children(node, mib[i]);
} else { } else {
const ctl_node_t *inode; const ctl_indexed_node_t *inode;
/* Indexed element. */ /* Indexed element. */
inode = &node->u.named.children[0]; inode = ctl_indexed_node(node->children);
node = inode->u.indexed.index(mib, miblen, mib[i]); node = inode->index(mib, miblen, mib[i]);
if (node == NULL) { if (node == NULL) {
ret = ENOENT; ret = ENOENT;
goto label_return; goto label_return;
@ -770,12 +804,12 @@ ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
} }
/* Call the ctl function. */ /* Call the ctl function. */
if (node->ctl == NULL) { if (node && node->ctl)
ret = node->ctl(mib, miblen, oldp, oldlenp, newp, newlen);
else {
/* Partial MIB. */ /* Partial MIB. */
ret = ENOENT; ret = ENOENT;
goto label_return;
} }
ret = node->ctl(mib, miblen, oldp, oldlenp, newp, newlen);
label_return: label_return:
return(ret); return(ret);
@ -799,22 +833,17 @@ ctl_boot(void)
#define READONLY() do { \ #define READONLY() do { \
if (newp != NULL || newlen != 0) { \ if (newp != NULL || newlen != 0) { \
ret = EPERM; \ ret = EPERM; \
goto label_return; \ goto label_return; \
} \ } \
} while (0) } while (0)
#define WRITEONLY() do { \ #define WRITEONLY() do { \
if (oldp != NULL || oldlenp != NULL) { \ if (oldp != NULL || oldlenp != NULL) { \
ret = EPERM; \ ret = EPERM; \
goto label_return; \ goto label_return; \
} \ } \
} while (0) } while (0)
#define VOID() do { \
READONLY(); \
WRITEONLY(); \
} while (0)
#define READ(v, t) do { \ #define READ(v, t) do { \
if (oldp != NULL && oldlenp != NULL) { \ if (oldp != NULL && oldlenp != NULL) { \
if (*oldlenp != sizeof(t)) { \ if (*oldlenp != sizeof(t)) { \
@ -822,7 +851,7 @@ ctl_boot(void)
? sizeof(t) : *oldlenp; \ ? sizeof(t) : *oldlenp; \
memcpy(oldp, (void *)&v, copylen); \ memcpy(oldp, (void *)&v, copylen); \
ret = EINVAL; \ ret = EINVAL; \
goto label_return; \ goto label_return; \
} else \ } else \
*(t *)oldp = v; \ *(t *)oldp = v; \
} \ } \
@ -832,7 +861,7 @@ ctl_boot(void)
if (newp != NULL) { \ if (newp != NULL) { \
if (newlen != sizeof(t)) { \ if (newlen != sizeof(t)) { \
ret = EINVAL; \ ret = EINVAL; \
goto label_return; \ goto label_return; \
} \ } \
v = *(t *)newp; \ v = *(t *)newp; \
} \ } \
@ -859,7 +888,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
READ(oldval, t); \ READ(oldval, t); \
\ \
ret = 0; \ ret = 0; \
label_return: \ label_return: \
if (l) \ if (l) \
malloc_mutex_unlock(&ctl_mtx); \ malloc_mutex_unlock(&ctl_mtx); \
return (ret); \ return (ret); \
@ -881,7 +910,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
READ(oldval, t); \ READ(oldval, t); \
\ \
ret = 0; \ ret = 0; \
label_return: \ label_return: \
malloc_mutex_unlock(&ctl_mtx); \ malloc_mutex_unlock(&ctl_mtx); \
return (ret); \ return (ret); \
} }
@ -900,7 +929,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
READ(oldval, t); \ READ(oldval, t); \
\ \
ret = 0; \ ret = 0; \
label_return: \ label_return: \
malloc_mutex_unlock(&ctl_mtx); \ malloc_mutex_unlock(&ctl_mtx); \
return (ret); \ return (ret); \
} }
@ -924,7 +953,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
READ(oldval, t); \ READ(oldval, t); \
\ \
ret = 0; \ ret = 0; \
label_return: \ label_return: \
return (ret); \ return (ret); \
} }
@ -941,7 +970,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
READ(oldval, t); \ READ(oldval, t); \
\ \
ret = 0; \ ret = 0; \
label_return: \ label_return: \
return (ret); \ return (ret); \
} }
@ -958,7 +987,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \
READ(oldval, bool); \ READ(oldval, bool); \
\ \
ret = 0; \ ret = 0; \
label_return: \ label_return: \
return (ret); \ return (ret); \
} }
@ -972,9 +1001,8 @@ epoch_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
uint64_t newval; uint64_t newval;
malloc_mutex_lock(&ctl_mtx); malloc_mutex_lock(&ctl_mtx);
newval = 0;
WRITE(newval, uint64_t); WRITE(newval, uint64_t);
if (newval != 0) if (newp != NULL)
ctl_refresh(); ctl_refresh();
READ(ctl_epoch, uint64_t); READ(ctl_epoch, uint64_t);
@ -1018,7 +1046,8 @@ thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp,
if (config_tcache == false) if (config_tcache == false)
return (ENOENT); return (ENOENT);
VOID(); READONLY();
WRITEONLY();
tcache_flush(); tcache_flush();
@ -1091,6 +1120,7 @@ CTL_RO_BOOL_CONFIG_GEN(config_debug)
CTL_RO_BOOL_CONFIG_GEN(config_dss) CTL_RO_BOOL_CONFIG_GEN(config_dss)
CTL_RO_BOOL_CONFIG_GEN(config_fill) CTL_RO_BOOL_CONFIG_GEN(config_fill)
CTL_RO_BOOL_CONFIG_GEN(config_lazy_lock) CTL_RO_BOOL_CONFIG_GEN(config_lazy_lock)
CTL_RO_BOOL_CONFIG_GEN(config_mremap)
CTL_RO_BOOL_CONFIG_GEN(config_munmap) CTL_RO_BOOL_CONFIG_GEN(config_munmap)
CTL_RO_BOOL_CONFIG_GEN(config_prof) CTL_RO_BOOL_CONFIG_GEN(config_prof)
CTL_RO_BOOL_CONFIG_GEN(config_prof_libgcc) CTL_RO_BOOL_CONFIG_GEN(config_prof_libgcc)
@ -1133,7 +1163,7 @@ CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool)
CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t)
CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t)
CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t) CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t)
const ctl_node_t * const ctl_named_node_t *
arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i) arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i)
{ {
@ -1143,7 +1173,7 @@ arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i)
} }
CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << LG_PAGE), size_t) CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << LG_PAGE), size_t)
const ctl_node_t * const ctl_named_node_t *
arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i)
{ {
@ -1201,7 +1231,7 @@ arenas_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
ret = EFAULT; ret = EFAULT;
goto label_return; goto label_return;
} else { } else {
arena_t *tarenas[narenas]; VARIABLE_ARRAY(arena_t *, tarenas, narenas);
malloc_mutex_lock(&arenas_lock); malloc_mutex_lock(&arenas_lock);
memcpy(tarenas, arenas, sizeof(arena_t *) * narenas); memcpy(tarenas, arenas, sizeof(arena_t *) * narenas);
@ -1326,7 +1356,7 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreruns,
CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curruns, CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curruns,
ctl_stats.arenas[mib[2]].bstats[mib[4]].curruns, size_t) ctl_stats.arenas[mib[2]].bstats[mib[4]].curruns, size_t)
const ctl_node_t * const ctl_named_node_t *
stats_arenas_i_bins_j_index(const size_t *mib, size_t miblen, size_t j) stats_arenas_i_bins_j_index(const size_t *mib, size_t miblen, size_t j)
{ {
@ -1344,7 +1374,7 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_nrequests,
CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_curruns, CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_curruns,
ctl_stats.arenas[mib[2]].lstats[mib[4]].curruns, size_t) ctl_stats.arenas[mib[2]].lstats[mib[4]].curruns, size_t)
const ctl_node_t * const ctl_named_node_t *
stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j) stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j)
{ {
@ -1365,10 +1395,10 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise,
CTL_RO_CGEN(config_stats, stats_arenas_i_purged, CTL_RO_CGEN(config_stats, stats_arenas_i_purged,
ctl_stats.arenas[mib[2]].astats.purged, uint64_t) ctl_stats.arenas[mib[2]].astats.purged, uint64_t)
const ctl_node_t * const ctl_named_node_t *
stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i) stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i)
{ {
const ctl_node_t * ret; const ctl_named_node_t * ret;
malloc_mutex_lock(&ctl_mtx); malloc_mutex_lock(&ctl_mtx);
if (ctl_stats.arenas[i].initialized == false) { if (ctl_stats.arenas[i].initialized == false) {

4
contrib/jemalloc/src/huge.c
View File

@ -140,11 +140,11 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
*/ */
copysize = (size < oldsize) ? size : oldsize; copysize = (size < oldsize) ? size : oldsize;
#ifdef JEMALLOC_MREMAP
/* /*
* Use mremap(2) if this is a huge-->huge reallocation, and neither the * Use mremap(2) if this is a huge-->huge reallocation, and neither the
* source nor the destination are in dss. * source nor the destination are in dss.
*/ */
#ifdef JEMALLOC_MREMAP_FIXED
if (oldsize >= chunksize && (config_dss == false || (chunk_in_dss(ptr) if (oldsize >= chunksize && (config_dss == false || (chunk_in_dss(ptr)
== false && chunk_in_dss(ret) == false))) { == false && chunk_in_dss(ret) == false))) {
size_t newsize = huge_salloc(ret); size_t newsize = huge_salloc(ret);
@ -168,7 +168,7 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
*/ */
char buf[BUFERROR_BUF]; char buf[BUFERROR_BUF];
buferror(errno, buf, sizeof(buf)); buferror(buf, sizeof(buf));
malloc_printf("<jemalloc>: Error in mremap(): %s\n", malloc_printf("<jemalloc>: Error in mremap(): %s\n",
buf); buf);
if (opt_abort) if (opt_abort)

122
contrib/jemalloc/src/jemalloc.c
View File

@ -13,7 +13,7 @@ const char *__malloc_options_1_0 = NULL;
__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0); __sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
/* Runtime configuration options. */ /* Runtime configuration options. */
const char *je_malloc_conf JEMALLOC_ATTR(visibility("default")); const char *je_malloc_conf;
#ifdef JEMALLOC_DEBUG #ifdef JEMALLOC_DEBUG
bool opt_abort = true; bool opt_abort = true;
# ifdef JEMALLOC_FILL # ifdef JEMALLOC_FILL
@ -56,7 +56,26 @@ static bool malloc_initializer = NO_INITIALIZER;
#endif #endif
/* Used to avoid initialization races. */ /* Used to avoid initialization races. */
#ifdef _WIN32
static malloc_mutex_t init_lock;
JEMALLOC_ATTR(constructor)
static void WINAPI
_init_init_lock(void)
{
malloc_mutex_init(&init_lock);
}
#ifdef _MSC_VER
# pragma section(".CRT$XCU", read)
JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used)
static const void (WINAPI *init_init_lock)(void) = _init_init_lock;
#endif
#else
static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
#endif
typedef struct { typedef struct {
void *p; /* Input pointer (as in realloc(p, s)). */ void *p; /* Input pointer (as in realloc(p, s)). */
@ -233,11 +252,17 @@ malloc_ncpus(void)
unsigned ret; unsigned ret;
long result; long result;
#ifdef _WIN32
SYSTEM_INFO si;
GetSystemInfo(&si);
result = si.dwNumberOfProcessors;
#else
result = sysconf(_SC_NPROCESSORS_ONLN); result = sysconf(_SC_NPROCESSORS_ONLN);
if (result == -1) { if (result == -1) {
/* Error. */ /* Error. */
ret = 1; ret = 1;
} }
#endif
ret = (unsigned)result; ret = (unsigned)result;
return (ret); return (ret);
@ -373,13 +398,14 @@ malloc_conf_init(void)
} }
break; break;
case 1: { case 1: {
#ifndef _WIN32
int linklen; int linklen;
const char *linkname = const char *linkname =
#ifdef JEMALLOC_PREFIX # ifdef JEMALLOC_PREFIX
"/etc/"JEMALLOC_PREFIX"malloc.conf" "/etc/"JEMALLOC_PREFIX"malloc.conf"
#else # else
"/etc/malloc.conf" "/etc/malloc.conf"
#endif # endif
; ;
if ((linklen = readlink(linkname, buf, if ((linklen = readlink(linkname, buf,
@ -390,7 +416,9 @@ malloc_conf_init(void)
*/ */
buf[linklen] = '\0'; buf[linklen] = '\0';
opts = buf; opts = buf;
} else { } else
#endif
{
/* No configuration specified. */ /* No configuration specified. */
buf[0] = '\0'; buf[0] = '\0';
opts = buf; opts = buf;
@ -456,9 +484,9 @@ malloc_conf_init(void)
uintmax_t um; \ uintmax_t um; \
char *end; \ char *end; \
\ \
errno = 0; \ set_errno(0); \
um = malloc_strtoumax(v, &end, 0); \ um = malloc_strtoumax(v, &end, 0); \
if (errno != 0 || (uintptr_t)end - \ if (get_errno() != 0 || (uintptr_t)end -\
(uintptr_t)v != vlen) { \ (uintptr_t)v != vlen) { \
malloc_conf_error( \ malloc_conf_error( \
"Invalid conf value", \ "Invalid conf value", \
@ -477,9 +505,9 @@ malloc_conf_init(void)
long l; \ long l; \
char *end; \ char *end; \
\ \
errno = 0; \ set_errno(0); \
l = strtol(v, &end, 0); \ l = strtol(v, &end, 0); \
if (errno != 0 || (uintptr_t)end - \ if (get_errno() != 0 || (uintptr_t)end -\
(uintptr_t)v != vlen) { \ (uintptr_t)v != vlen) { \
malloc_conf_error( \ malloc_conf_error( \
"Invalid conf value", \ "Invalid conf value", \
@ -615,7 +643,8 @@ malloc_init_hard(void)
malloc_conf_init(); malloc_conf_init();
#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE)) #if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \
&& !defined(_WIN32))
/* Register fork handlers. */ /* Register fork handlers. */
if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent,
jemalloc_postfork_child) != 0) { jemalloc_postfork_child) != 0) {
@ -770,13 +799,11 @@ malloc_init_hard(void)
* Begin malloc(3)-compatible functions. * Begin malloc(3)-compatible functions.
*/ */
JEMALLOC_ATTR(malloc)
JEMALLOC_ATTR(visibility("default"))
void * void *
je_malloc(size_t size) je_malloc(size_t size)
{ {
void *ret; void *ret;
size_t usize; size_t usize JEMALLOC_CC_SILENCE_INIT(0);
prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL);
if (malloc_init()) { if (malloc_init()) {
@ -814,7 +841,7 @@ je_malloc(size_t size)
"out of memory\n"); "out of memory\n");
abort(); abort();
} }
errno = ENOMEM; set_errno(ENOMEM);
} }
if (config_prof && opt_prof && ret != NULL) if (config_prof && opt_prof && ret != NULL)
prof_malloc(ret, usize, cnt); prof_malloc(ret, usize, cnt);
@ -921,8 +948,6 @@ imemalign(void **memptr, size_t alignment, size_t size,
return (ret); return (ret);
} }
JEMALLOC_ATTR(nonnull(1))
JEMALLOC_ATTR(visibility("default"))
int int
je_posix_memalign(void **memptr, size_t alignment, size_t size) je_posix_memalign(void **memptr, size_t alignment, size_t size)
{ {
@ -932,8 +957,6 @@ je_posix_memalign(void **memptr, size_t alignment, size_t size)
return (ret); return (ret);
} }
JEMALLOC_ATTR(malloc)
JEMALLOC_ATTR(visibility("default"))
void * void *
je_aligned_alloc(size_t alignment, size_t size) je_aligned_alloc(size_t alignment, size_t size)
{ {
@ -942,21 +965,19 @@ je_aligned_alloc(size_t alignment, size_t size)
if ((err = imemalign(&ret, alignment, size, 1)) != 0) { if ((err = imemalign(&ret, alignment, size, 1)) != 0) {
ret = NULL; ret = NULL;
errno = err; set_errno(err);
} }
JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof),
false); false);
return (ret); return (ret);
} }
JEMALLOC_ATTR(malloc)
JEMALLOC_ATTR(visibility("default"))
void * void *
je_calloc(size_t num, size_t size) je_calloc(size_t num, size_t size)
{ {
void *ret; void *ret;
size_t num_size; size_t num_size;
size_t usize; size_t usize JEMALLOC_CC_SILENCE_INIT(0);
prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL);
if (malloc_init()) { if (malloc_init()) {
@ -1012,7 +1033,7 @@ je_calloc(size_t num, size_t size)
"memory\n"); "memory\n");
abort(); abort();
} }
errno = ENOMEM; set_errno(ENOMEM);
} }
if (config_prof && opt_prof && ret != NULL) if (config_prof && opt_prof && ret != NULL)
@ -1026,12 +1047,11 @@ je_calloc(size_t num, size_t size)
return (ret); return (ret);
} }
JEMALLOC_ATTR(visibility("default"))
void * void *
je_realloc(void *ptr, size_t size) je_realloc(void *ptr, size_t size)
{ {
void *ret; void *ret;
size_t usize; size_t usize JEMALLOC_CC_SILENCE_INIT(0);
size_t old_size = 0; size_t old_size = 0;
size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0);
prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL);
@ -1113,7 +1133,7 @@ je_realloc(void *ptr, size_t size)
"out of memory\n"); "out of memory\n");
abort(); abort();
} }
errno = ENOMEM; set_errno(ENOMEM);
} }
} else { } else {
/* realloc(NULL, size) is equivalent to malloc(size). */ /* realloc(NULL, size) is equivalent to malloc(size). */
@ -1155,7 +1175,7 @@ je_realloc(void *ptr, size_t size)
"out of memory\n"); "out of memory\n");
abort(); abort();
} }
errno = ENOMEM; set_errno(ENOMEM);
} }
} }
@ -1174,7 +1194,6 @@ je_realloc(void *ptr, size_t size)
return (ret); return (ret);
} }
JEMALLOC_ATTR(visibility("default"))
void void
je_free(void *ptr) je_free(void *ptr)
{ {
@ -1209,8 +1228,6 @@ je_free(void *ptr)
*/ */
#ifdef JEMALLOC_OVERRIDE_MEMALIGN #ifdef JEMALLOC_OVERRIDE_MEMALIGN
JEMALLOC_ATTR(malloc)
JEMALLOC_ATTR(visibility("default"))
void * void *
je_memalign(size_t alignment, size_t size) je_memalign(size_t alignment, size_t size)
{ {
@ -1222,8 +1239,6 @@ je_memalign(size_t alignment, size_t size)
#endif #endif
#ifdef JEMALLOC_OVERRIDE_VALLOC #ifdef JEMALLOC_OVERRIDE_VALLOC
JEMALLOC_ATTR(malloc)
JEMALLOC_ATTR(visibility("default"))
void * void *
je_valloc(size_t size) je_valloc(size_t size)
{ {
@ -1252,17 +1267,12 @@ je_valloc(size_t size)
* passed an extra argument for the caller return address, which will be * passed an extra argument for the caller return address, which will be
* ignored. * ignored.
*/ */
JEMALLOC_ATTR(visibility("default")) JEMALLOC_EXPORT void (* const __free_hook)(void *ptr) = je_free;
void (* const __free_hook)(void *ptr) = je_free; JEMALLOC_EXPORT void *(* const __malloc_hook)(size_t size) = je_malloc;
JEMALLOC_EXPORT void *(* const __realloc_hook)(void *ptr, size_t size) =
JEMALLOC_ATTR(visibility("default")) je_realloc;
void *(* const __malloc_hook)(size_t size) = je_malloc; JEMALLOC_EXPORT void *(* const __memalign_hook)(size_t alignment, size_t size) =
je_memalign;
JEMALLOC_ATTR(visibility("default"))
void *(* const __realloc_hook)(void *ptr, size_t size) = je_realloc;
JEMALLOC_ATTR(visibility("default"))
void *(* const __memalign_hook)(size_t alignment, size_t size) = je_memalign;
#endif #endif
/* /*
@ -1273,7 +1283,6 @@ void *(* const __memalign_hook)(size_t alignment, size_t size) = je_memalign;
* Begin non-standard functions. * Begin non-standard functions.
*/ */
JEMALLOC_ATTR(visibility("default"))
size_t size_t
je_malloc_usable_size(const void *ptr) je_malloc_usable_size(const void *ptr)
{ {
@ -1289,7 +1298,6 @@ je_malloc_usable_size(const void *ptr)
return (ret); return (ret);
} }
JEMALLOC_ATTR(visibility("default"))
void void
je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
const char *opts) const char *opts)
@ -1298,7 +1306,6 @@ je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
stats_print(write_cb, cbopaque, opts); stats_print(write_cb, cbopaque, opts);
} }
JEMALLOC_ATTR(visibility("default"))
int int
je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
size_t newlen) size_t newlen)
@ -1310,7 +1317,6 @@ je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp,
return (ctl_byname(name, oldp, oldlenp, newp, newlen)); return (ctl_byname(name, oldp, oldlenp, newp, newlen));
} }
JEMALLOC_ATTR(visibility("default"))
int int
je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp)
{ {
@ -1321,7 +1327,6 @@ je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp)
return (ctl_nametomib(name, mibp, miblenp)); return (ctl_nametomib(name, mibp, miblenp));
} }
JEMALLOC_ATTR(visibility("default"))
int int
je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp,
void *newp, size_t newlen) void *newp, size_t newlen)
@ -1357,8 +1362,6 @@ iallocm(size_t usize, size_t alignment, bool zero)
return (imalloc(usize)); return (imalloc(usize));
} }
JEMALLOC_ATTR(nonnull(1))
JEMALLOC_ATTR(visibility("default"))
int int
je_allocm(void **ptr, size_t *rsize, size_t size, int flags) je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
{ {
@ -1367,7 +1370,6 @@ je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK)
& (SIZE_T_MAX-1)); & (SIZE_T_MAX-1));
bool zero = flags & ALLOCM_ZERO; bool zero = flags & ALLOCM_ZERO;
prof_thr_cnt_t *cnt;
assert(ptr != NULL); assert(ptr != NULL);
assert(size != 0); assert(size != 0);
@ -1380,6 +1382,8 @@ je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
goto label_oom; goto label_oom;
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
prof_thr_cnt_t *cnt;
PROF_ALLOC_PREP(1, usize, cnt); PROF_ALLOC_PREP(1, usize, cnt);
if (cnt == NULL) if (cnt == NULL)
goto label_oom; goto label_oom;
@ -1426,8 +1430,6 @@ je_allocm(void **ptr, size_t *rsize, size_t size, int flags)
return (ALLOCM_ERR_OOM); return (ALLOCM_ERR_OOM);
} }
JEMALLOC_ATTR(nonnull(1))
JEMALLOC_ATTR(visibility("default"))
int int
je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
{ {
@ -1439,7 +1441,6 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
& (SIZE_T_MAX-1)); & (SIZE_T_MAX-1));
bool zero = flags & ALLOCM_ZERO; bool zero = flags & ALLOCM_ZERO;
bool no_move = flags & ALLOCM_NO_MOVE; bool no_move = flags & ALLOCM_NO_MOVE;
prof_thr_cnt_t *cnt;
assert(ptr != NULL); assert(ptr != NULL);
assert(*ptr != NULL); assert(*ptr != NULL);
@ -1449,6 +1450,8 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
p = *ptr; p = *ptr;
if (config_prof && opt_prof) { if (config_prof && opt_prof) {
prof_thr_cnt_t *cnt;
/* /*
* usize isn't knowable before iralloc() returns when extra is * usize isn't knowable before iralloc() returns when extra is
* non-zero. Therefore, compute its maximum possible value and * non-zero. Therefore, compute its maximum possible value and
@ -1536,8 +1539,6 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags)
return (ALLOCM_ERR_OOM); return (ALLOCM_ERR_OOM);
} }
JEMALLOC_ATTR(nonnull(1))
JEMALLOC_ATTR(visibility("default"))
int int
je_sallocm(const void *ptr, size_t *rsize, int flags) je_sallocm(const void *ptr, size_t *rsize, int flags)
{ {
@ -1557,8 +1558,6 @@ je_sallocm(const void *ptr, size_t *rsize, int flags)
return (ALLOCM_SUCCESS); return (ALLOCM_SUCCESS);
} }
JEMALLOC_ATTR(nonnull(1))
JEMALLOC_ATTR(visibility("default"))
int int
je_dallocm(void *ptr, int flags) je_dallocm(void *ptr, int flags)
{ {
@ -1586,7 +1585,6 @@ je_dallocm(void *ptr, int flags)
return (ALLOCM_SUCCESS); return (ALLOCM_SUCCESS);
} }
JEMALLOC_ATTR(visibility("default"))
int int
je_nallocm(size_t *rsize, size_t size, int flags) je_nallocm(size_t *rsize, size_t size, int flags)
{ {
@ -1622,8 +1620,7 @@ je_nallocm(size_t *rsize, size_t size, int flags)
void void
jemalloc_prefork(void) jemalloc_prefork(void)
#else #else
JEMALLOC_ATTR(visibility("default")) JEMALLOC_EXPORT void
void
_malloc_prefork(void) _malloc_prefork(void)
#endif #endif
{ {
@ -1644,8 +1641,7 @@ _malloc_prefork(void)
void void
jemalloc_postfork_parent(void) jemalloc_postfork_parent(void)
#else #else
JEMALLOC_ATTR(visibility("default")) JEMALLOC_EXPORT void
void
_malloc_postfork(void) _malloc_postfork(void)
#endif #endif
{ {

21
contrib/jemalloc/src/mutex.c
View File

@ -1,10 +1,14 @@
#define JEMALLOC_MUTEX_C_ #define JEMALLOC_MUTEX_C_
#include "jemalloc/internal/jemalloc_internal.h" #include "jemalloc/internal/jemalloc_internal.h"
#ifdef JEMALLOC_LAZY_LOCK #if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32)
#include <dlfcn.h> #include <dlfcn.h>
#endif #endif
#ifndef _CRT_SPINCOUNT
#define _CRT_SPINCOUNT 4000
#endif
/******************************************************************************/ /******************************************************************************/
/* Data. */ /* Data. */
@ -16,7 +20,7 @@ static bool postpone_init = true;
static malloc_mutex_t *postponed_mutexes = NULL; static malloc_mutex_t *postponed_mutexes = NULL;
#endif #endif
#ifdef JEMALLOC_LAZY_LOCK #if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32)
static void pthread_create_once(void); static void pthread_create_once(void);
#endif #endif
@ -26,7 +30,7 @@ static void pthread_create_once(void);
* process goes multi-threaded. * process goes multi-threaded.
*/ */
#ifdef JEMALLOC_LAZY_LOCK #if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32)
static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *,
void *(*)(void *), void *__restrict); void *(*)(void *), void *__restrict);
@ -44,8 +48,7 @@ pthread_create_once(void)
isthreaded = true; isthreaded = true;
} }
JEMALLOC_ATTR(visibility("default")) JEMALLOC_EXPORT int
int
pthread_create(pthread_t *__restrict thread, pthread_create(pthread_t *__restrict thread,
const pthread_attr_t *__restrict attr, void *(*start_routine)(void *), const pthread_attr_t *__restrict attr, void *(*start_routine)(void *),
void *__restrict arg) void *__restrict arg)
@ -79,7 +82,12 @@ _pthread_mutex_init_calloc_cb_stub(pthread_mutex_t *mutex,
bool bool
malloc_mutex_init(malloc_mutex_t *mutex) malloc_mutex_init(malloc_mutex_t *mutex)
{ {
#ifdef JEMALLOC_OSSPIN
#ifdef _WIN32
if (!InitializeCriticalSectionAndSpinCount(&mutex->lock,
_CRT_SPINCOUNT))
return (true);
#elif (defined(JEMALLOC_OSSPIN))
mutex->lock = 0; mutex->lock = 0;
#elif (defined(JEMALLOC_MUTEX_INIT_CB)) #elif (defined(JEMALLOC_MUTEX_INIT_CB))
if (postpone_init) { if (postpone_init) {
@ -101,7 +109,6 @@ malloc_mutex_init(malloc_mutex_t *mutex)
return (true); return (true);
} }
pthread_mutexattr_destroy(&attr); pthread_mutexattr_destroy(&attr);
#endif #endif
return (false); return (false);
} }

215
contrib/jemalloc/src/prof.c
View File

@ -64,11 +64,6 @@ static int prof_dump_fd;
/* Do not dump any profiles until bootstrapping is complete. */ /* Do not dump any profiles until bootstrapping is complete. */
static bool prof_booted = false; static bool prof_booted = false;
static malloc_mutex_t enq_mtx;
static bool enq;
static bool enq_idump;
static bool enq_gdump;
/******************************************************************************/ /******************************************************************************/
/* Function prototypes for non-inline static functions. */ /* Function prototypes for non-inline static functions. */
@ -148,20 +143,19 @@ bt_dup(prof_bt_t *bt)
} }
static inline void static inline void
prof_enter(void) prof_enter(prof_tdata_t *prof_tdata)
{ {
cassert(config_prof); cassert(config_prof);
malloc_mutex_lock(&enq_mtx); assert(prof_tdata->enq == false);
enq = true; prof_tdata->enq = true;
malloc_mutex_unlock(&enq_mtx);
malloc_mutex_lock(&bt2ctx_mtx); malloc_mutex_lock(&bt2ctx_mtx);
} }
static inline void static inline void
prof_leave(void) prof_leave(prof_tdata_t *prof_tdata)
{ {
bool idump, gdump; bool idump, gdump;
@ -169,13 +163,12 @@ prof_leave(void)
malloc_mutex_unlock(&bt2ctx_mtx); malloc_mutex_unlock(&bt2ctx_mtx);
malloc_mutex_lock(&enq_mtx); assert(prof_tdata->enq);
enq = false; prof_tdata->enq = false;
idump = enq_idump; idump = prof_tdata->enq_idump;
enq_idump = false; prof_tdata->enq_idump = false;
gdump = enq_gdump; gdump = prof_tdata->enq_gdump;
enq_gdump = false; prof_tdata->enq_gdump = false;
malloc_mutex_unlock(&enq_mtx);
if (idump) if (idump)
prof_idump(); prof_idump();
@ -446,12 +439,9 @@ prof_lookup(prof_bt_t *bt)
cassert(config_prof); cassert(config_prof);
prof_tdata = *prof_tdata_tsd_get(); prof_tdata = prof_tdata_get();
if (prof_tdata == NULL) { if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
prof_tdata = prof_tdata_init(); return (NULL);
if (prof_tdata == NULL)
return (NULL);
}
if (ckh_search(&prof_tdata->bt2cnt, bt, NULL, &ret.v)) { if (ckh_search(&prof_tdata->bt2cnt, bt, NULL, &ret.v)) {
union { union {
@ -468,52 +458,48 @@ prof_lookup(prof_bt_t *bt)
* This thread's cache lacks bt. Look for it in the global * This thread's cache lacks bt. Look for it in the global
* cache. * cache.
*/ */
prof_enter(); prof_enter(prof_tdata);
if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) { if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) {
/* bt has never been seen before. Insert it. */ /* bt has never been seen before. Insert it. */
ctx.v = imalloc(sizeof(prof_ctx_t)); ctx.v = imalloc(sizeof(prof_ctx_t));
if (ctx.v == NULL) { if (ctx.v == NULL) {
prof_leave(); prof_leave(prof_tdata);
return (NULL); return (NULL);
} }
btkey.p = bt_dup(bt); btkey.p = bt_dup(bt);
if (btkey.v == NULL) { if (btkey.v == NULL) {
prof_leave(); prof_leave(prof_tdata);
idalloc(ctx.v); idalloc(ctx.v);
return (NULL); return (NULL);
} }
ctx.p->bt = btkey.p; ctx.p->bt = btkey.p;
ctx.p->lock = prof_ctx_mutex_choose(); ctx.p->lock = prof_ctx_mutex_choose();
/*
* Set nlimbo to 1, in order to avoid a race condition
* with prof_ctx_merge()/prof_ctx_destroy().
*/
ctx.p->nlimbo = 1;
memset(&ctx.p->cnt_merged, 0, sizeof(prof_cnt_t)); memset(&ctx.p->cnt_merged, 0, sizeof(prof_cnt_t));
ql_new(&ctx.p->cnts_ql); ql_new(&ctx.p->cnts_ql);
if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) { if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) {
/* OOM. */ /* OOM. */
prof_leave(); prof_leave(prof_tdata);
idalloc(btkey.v); idalloc(btkey.v);
idalloc(ctx.v); idalloc(ctx.v);
return (NULL); return (NULL);
} }
/*
* Artificially raise curobjs, in order to avoid a race
* condition with prof_ctx_merge()/prof_ctx_destroy().
*
* No locking is necessary for ctx here because no other
* threads have had the opportunity to fetch it from
* bt2ctx yet.
*/
ctx.p->cnt_merged.curobjs++;
new_ctx = true; new_ctx = true;
} else { } else {
/* /*
* Artificially raise curobjs, in order to avoid a race * Increment nlimbo, in order to avoid a race condition
* condition with prof_ctx_merge()/prof_ctx_destroy(). * with prof_ctx_merge()/prof_ctx_destroy().
*/ */
malloc_mutex_lock(ctx.p->lock); malloc_mutex_lock(ctx.p->lock);
ctx.p->cnt_merged.curobjs++; ctx.p->nlimbo++;
malloc_mutex_unlock(ctx.p->lock); malloc_mutex_unlock(ctx.p->lock);
new_ctx = false; new_ctx = false;
} }
prof_leave(); prof_leave(prof_tdata);
/* Link a prof_thd_cnt_t into ctx for this thread. */ /* Link a prof_thd_cnt_t into ctx for this thread. */
if (ckh_count(&prof_tdata->bt2cnt) == PROF_TCMAX) { if (ckh_count(&prof_tdata->bt2cnt) == PROF_TCMAX) {
@ -555,7 +541,7 @@ prof_lookup(prof_bt_t *bt)
ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link);
malloc_mutex_lock(ctx.p->lock); malloc_mutex_lock(ctx.p->lock);
ql_tail_insert(&ctx.p->cnts_ql, ret.p, cnts_link); ql_tail_insert(&ctx.p->cnts_ql, ret.p, cnts_link);
ctx.p->cnt_merged.curobjs--; ctx.p->nlimbo--;
malloc_mutex_unlock(ctx.p->lock); malloc_mutex_unlock(ctx.p->lock);
} else { } else {
/* Move ret to the front of the LRU. */ /* Move ret to the front of the LRU. */
@ -688,26 +674,30 @@ prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx)
static void static void
prof_ctx_destroy(prof_ctx_t *ctx) prof_ctx_destroy(prof_ctx_t *ctx)
{ {
prof_tdata_t *prof_tdata;
cassert(config_prof); cassert(config_prof);
/* /*
* Check that ctx is still unused by any thread cache before destroying * Check that ctx is still unused by any thread cache before destroying
* it. prof_lookup() artificially raises ctx->cnt_merge.curobjs in * it. prof_lookup() increments ctx->nlimbo in order to avoid a race
* order to avoid a race condition with this function, as does * condition with this function, as does prof_ctx_merge() in order to
* prof_ctx_merge() in order to avoid a race between the main body of * avoid a race between the main body of prof_ctx_merge() and entry
* prof_ctx_merge() and entry into this function. * into this function.
*/ */
prof_enter(); prof_tdata = *prof_tdata_tsd_get();
assert((uintptr_t)prof_tdata > (uintptr_t)PROF_TDATA_STATE_MAX);
prof_enter(prof_tdata);
malloc_mutex_lock(ctx->lock); malloc_mutex_lock(ctx->lock);
if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 1) { if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 0 &&
ctx->nlimbo == 1) {
assert(ctx->cnt_merged.curbytes == 0); assert(ctx->cnt_merged.curbytes == 0);
assert(ctx->cnt_merged.accumobjs == 0); assert(ctx->cnt_merged.accumobjs == 0);
assert(ctx->cnt_merged.accumbytes == 0); assert(ctx->cnt_merged.accumbytes == 0);
/* Remove ctx from bt2ctx. */ /* Remove ctx from bt2ctx. */
if (ckh_remove(&bt2ctx, ctx->bt, NULL, NULL)) if (ckh_remove(&bt2ctx, ctx->bt, NULL, NULL))
assert(false); assert(false);
prof_leave(); prof_leave(prof_tdata);
/* Destroy ctx. */ /* Destroy ctx. */
malloc_mutex_unlock(ctx->lock); malloc_mutex_unlock(ctx->lock);
bt_destroy(ctx->bt); bt_destroy(ctx->bt);
@ -717,9 +707,9 @@ prof_ctx_destroy(prof_ctx_t *ctx)
* Compensate for increment in prof_ctx_merge() or * Compensate for increment in prof_ctx_merge() or
* prof_lookup(). * prof_lookup().
*/ */
ctx->cnt_merged.curobjs--; ctx->nlimbo--;
malloc_mutex_unlock(ctx->lock); malloc_mutex_unlock(ctx->lock);
prof_leave(); prof_leave(prof_tdata);
} }
} }
@ -738,12 +728,12 @@ prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt)
ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes; ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes;
ql_remove(&ctx->cnts_ql, cnt, cnts_link); ql_remove(&ctx->cnts_ql, cnt, cnts_link);
if (opt_prof_accum == false && ql_first(&ctx->cnts_ql) == NULL && if (opt_prof_accum == false && ql_first(&ctx->cnts_ql) == NULL &&
ctx->cnt_merged.curobjs == 0) { ctx->cnt_merged.curobjs == 0 && ctx->nlimbo == 0) {
/* /*
* Artificially raise ctx->cnt_merged.curobjs in order to keep * Increment ctx->nlimbo in order to keep another thread from
* another thread from winning the race to destroy ctx while * winning the race to destroy ctx while this one has ctx->lock
* this one has ctx->lock dropped. Without this, it would be * dropped. Without this, it would be possible for another
* possible for another thread to: * thread to:
* *
* 1) Sample an allocation associated with ctx. * 1) Sample an allocation associated with ctx.
* 2) Deallocate the sampled object. * 2) Deallocate the sampled object.
@ -752,7 +742,7 @@ prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt)
* The result would be that ctx no longer exists by the time * The result would be that ctx no longer exists by the time
* this thread accesses it in prof_ctx_destroy(). * this thread accesses it in prof_ctx_destroy().
*/ */
ctx->cnt_merged.curobjs++; ctx->nlimbo++;
destroy = true; destroy = true;
} else } else
destroy = false; destroy = false;
@ -768,7 +758,16 @@ prof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, prof_bt_t *bt)
cassert(config_prof); cassert(config_prof);
if (opt_prof_accum == false && ctx->cnt_summed.curobjs == 0) { /*
* Current statistics can sum to 0 as a result of unmerged per thread
* statistics. Additionally, interval- and growth-triggered dumps can
* occur between the time a ctx is created and when its statistics are
* filled in. Avoid dumping any ctx that is an artifact of either
* implementation detail.
*/
if ((opt_prof_accum == false && ctx->cnt_summed.curobjs == 0) ||
(opt_prof_accum && ctx->cnt_summed.accumobjs == 0)) {
assert(ctx->cnt_summed.curobjs == 0);
assert(ctx->cnt_summed.curbytes == 0); assert(ctx->cnt_summed.curbytes == 0);
assert(ctx->cnt_summed.accumobjs == 0); assert(ctx->cnt_summed.accumobjs == 0);
assert(ctx->cnt_summed.accumbytes == 0); assert(ctx->cnt_summed.accumbytes == 0);
@ -831,6 +830,7 @@ prof_dump_maps(bool propagate_err)
static bool static bool
prof_dump(bool propagate_err, const char *filename, bool leakcheck) prof_dump(bool propagate_err, const char *filename, bool leakcheck)
{ {
prof_tdata_t *prof_tdata;
prof_cnt_t cnt_all; prof_cnt_t cnt_all;
size_t tabind; size_t tabind;
union { union {
@ -845,7 +845,10 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck)
cassert(config_prof); cassert(config_prof);
prof_enter(); prof_tdata = prof_tdata_get();
if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
return (true);
prof_enter(prof_tdata);
prof_dump_fd = creat(filename, 0644); prof_dump_fd = creat(filename, 0644);
if (prof_dump_fd == -1) { if (prof_dump_fd == -1) {
if (propagate_err == false) { if (propagate_err == false) {
@ -896,7 +899,7 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck)
if (prof_flush(propagate_err)) if (prof_flush(propagate_err))
goto label_error; goto label_error;
close(prof_dump_fd); close(prof_dump_fd);
prof_leave(); prof_leave(prof_tdata);
if (leakcheck && cnt_all.curbytes != 0) { if (leakcheck && cnt_all.curbytes != 0) {
malloc_printf("<jemalloc>: Leak summary: %"PRId64" byte%s, %" malloc_printf("<jemalloc>: Leak summary: %"PRId64" byte%s, %"
@ -911,7 +914,7 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck)
return (false); return (false);
label_error: label_error:
prof_leave(); prof_leave(prof_tdata);
return (true); return (true);
} }
@ -933,6 +936,7 @@ prof_dump_filename(char *filename, char v, int64_t vseq)
"%s.%d.%"PRIu64".%c.heap", "%s.%d.%"PRIu64".%c.heap",
opt_prof_prefix, (int)getpid(), prof_dump_seq, v); opt_prof_prefix, (int)getpid(), prof_dump_seq, v);
} }
prof_dump_seq++;
} }
static void static void
@ -956,19 +960,24 @@ prof_fdump(void)
void void
prof_idump(void) prof_idump(void)
{ {
prof_tdata_t *prof_tdata;
char filename[PATH_MAX + 1]; char filename[PATH_MAX + 1];
cassert(config_prof); cassert(config_prof);
if (prof_booted == false) if (prof_booted == false)
return; return;
malloc_mutex_lock(&enq_mtx); /*
if (enq) { * Don't call prof_tdata_get() here, because it could cause recursive
enq_idump = true; * allocation.
malloc_mutex_unlock(&enq_mtx); */
prof_tdata = *prof_tdata_tsd_get();
if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
return;
if (prof_tdata->enq) {
prof_tdata->enq_idump = true;
return; return;
} }
malloc_mutex_unlock(&enq_mtx);
if (opt_prof_prefix[0] != '\0') { if (opt_prof_prefix[0] != '\0') {
malloc_mutex_lock(&prof_dump_seq_mtx); malloc_mutex_lock(&prof_dump_seq_mtx);
@ -1005,19 +1014,24 @@ prof_mdump(const char *filename)
void void
prof_gdump(void) prof_gdump(void)
{ {
prof_tdata_t *prof_tdata;
char filename[DUMP_FILENAME_BUFSIZE]; char filename[DUMP_FILENAME_BUFSIZE];
cassert(config_prof); cassert(config_prof);
if (prof_booted == false) if (prof_booted == false)
return; return;
malloc_mutex_lock(&enq_mtx); /*
if (enq) { * Don't call prof_tdata_get() here, because it could cause recursive
enq_gdump = true; * allocation.
malloc_mutex_unlock(&enq_mtx); */
prof_tdata = *prof_tdata_tsd_get();
if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)
return;
if (prof_tdata->enq) {
prof_tdata->enq_gdump = true;
return; return;
} }
malloc_mutex_unlock(&enq_mtx);
if (opt_prof_prefix[0] != '\0') { if (opt_prof_prefix[0] != '\0') {
malloc_mutex_lock(&prof_dump_seq_mtx); malloc_mutex_lock(&prof_dump_seq_mtx);
@ -1110,6 +1124,10 @@ prof_tdata_init(void)
prof_tdata->threshold = 0; prof_tdata->threshold = 0;
prof_tdata->accum = 0; prof_tdata->accum = 0;
prof_tdata->enq = false;
prof_tdata->enq_idump = false;
prof_tdata->enq_gdump = false;
prof_tdata_tsd_set(&prof_tdata); prof_tdata_tsd_set(&prof_tdata);
return (prof_tdata); return (prof_tdata);
@ -1123,24 +1141,41 @@ prof_tdata_cleanup(void *arg)
cassert(config_prof); cassert(config_prof);
/* if (prof_tdata == PROF_TDATA_STATE_REINCARNATED) {
* Delete the hash table. All of its contents can still be iterated /*
* over via the LRU. * Another destructor deallocated memory after this destructor
*/ * was called. Reset prof_tdata to PROF_TDATA_STATE_PURGATORY
ckh_delete(&prof_tdata->bt2cnt); * in order to receive another callback.
*/
/* Iteratively merge cnt's into the global stats and delete them. */ prof_tdata = PROF_TDATA_STATE_PURGATORY;
while ((cnt = ql_last(&prof_tdata->lru_ql, lru_link)) != NULL) { prof_tdata_tsd_set(&prof_tdata);
ql_remove(&prof_tdata->lru_ql, cnt, lru_link); } else if (prof_tdata == PROF_TDATA_STATE_PURGATORY) {
prof_ctx_merge(cnt->ctx, cnt); /*
idalloc(cnt); * The previous time this destructor was called, we set the key
* to PROF_TDATA_STATE_PURGATORY so that other destructors
* wouldn't cause re-creation of the prof_tdata. This time, do
* nothing, so that the destructor will not be called again.
*/
} else if (prof_tdata != NULL) {
/*
* Delete the hash table. All of its contents can still be
* iterated over via the LRU.
*/
ckh_delete(&prof_tdata->bt2cnt);
/*
* Iteratively merge cnt's into the global stats and delete
* them.
*/
while ((cnt = ql_last(&prof_tdata->lru_ql, lru_link)) != NULL) {
ql_remove(&prof_tdata->lru_ql, cnt, lru_link);
prof_ctx_merge(cnt->ctx, cnt);
idalloc(cnt);
}
idalloc(prof_tdata->vec);
idalloc(prof_tdata);
prof_tdata = PROF_TDATA_STATE_PURGATORY;
prof_tdata_tsd_set(&prof_tdata);
} }
idalloc(prof_tdata->vec);
idalloc(prof_tdata);
prof_tdata = NULL;
prof_tdata_tsd_set(&prof_tdata);
} }
void void
@ -1206,12 +1241,6 @@ prof_boot2(void)
if (malloc_mutex_init(&prof_dump_seq_mtx)) if (malloc_mutex_init(&prof_dump_seq_mtx))
return (true); return (true);
if (malloc_mutex_init(&enq_mtx))
return (true);
enq = false;
enq_idump = false;
enq_gdump = false;
if (atexit(prof_fdump) != 0) { if (atexit(prof_fdump) != 0) {
malloc_write("<jemalloc>: Error in atexit()\n"); malloc_write("<jemalloc>: Error in atexit()\n");
if (opt_abort) if (opt_abort)

101
contrib/jemalloc/src/quarantine.c
View File

@ -1,17 +1,31 @@
#include "jemalloc/internal/jemalloc_internal.h" #include "jemalloc/internal/jemalloc_internal.h"
/*
* quarantine pointers close to NULL are used to encode state information that
* is used for cleaning up during thread shutdown.
*/
#define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1)
#define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2)
#define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY
/******************************************************************************/ /******************************************************************************/
/* Data. */ /* Data. */
typedef struct quarantine_obj_s quarantine_obj_t;
typedef struct quarantine_s quarantine_t; typedef struct quarantine_s quarantine_t;
struct quarantine_obj_s {
void *ptr;
size_t usize;
};
struct quarantine_s { struct quarantine_s {
size_t curbytes; size_t curbytes;
size_t curobjs; size_t curobjs;
size_t first; size_t first;
#define LG_MAXOBJS_INIT 10 #define LG_MAXOBJS_INIT 10
size_t lg_maxobjs; size_t lg_maxobjs;
void *objs[1]; /* Dynamically sized ring buffer. */ quarantine_obj_t objs[1]; /* Dynamically sized ring buffer. */
}; };
static void quarantine_cleanup(void *arg); static void quarantine_cleanup(void *arg);
@ -35,7 +49,7 @@ quarantine_init(size_t lg_maxobjs)
quarantine_t *quarantine; quarantine_t *quarantine;
quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) + quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) +
((ZU(1) << lg_maxobjs) * sizeof(void *))); ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t)));
if (quarantine == NULL) if (quarantine == NULL)
return (NULL); return (NULL);
quarantine->curbytes = 0; quarantine->curbytes = 0;
@ -58,23 +72,22 @@ quarantine_grow(quarantine_t *quarantine)
return (quarantine); return (quarantine);
ret->curbytes = quarantine->curbytes; ret->curbytes = quarantine->curbytes;
if (quarantine->first + quarantine->curobjs < (ZU(1) << ret->curobjs = quarantine->curobjs;
if (quarantine->first + quarantine->curobjs <= (ZU(1) <<
quarantine->lg_maxobjs)) { quarantine->lg_maxobjs)) {
/* objs ring buffer data are contiguous. */ /* objs ring buffer data are contiguous. */
memcpy(ret->objs, &quarantine->objs[quarantine->first], memcpy(ret->objs, &quarantine->objs[quarantine->first],
quarantine->curobjs * sizeof(void *)); quarantine->curobjs * sizeof(quarantine_obj_t));
ret->curobjs = quarantine->curobjs;
} else { } else {
/* objs ring buffer data wrap around. */ /* objs ring buffer data wrap around. */
size_t ncopy = (ZU(1) << quarantine->lg_maxobjs) - size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) -
quarantine->first; quarantine->first;
memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy * size_t ncopy_b = quarantine->curobjs - ncopy_a;
sizeof(void *));
ret->curobjs = ncopy; memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a
if (quarantine->curobjs != 0) { * sizeof(quarantine_obj_t));
memcpy(&ret->objs[ret->curobjs], quarantine->objs, memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b *
quarantine->curobjs - ncopy); sizeof(quarantine_obj_t));
}
} }
return (ret); return (ret);
@ -85,10 +98,10 @@ quarantine_drain(quarantine_t *quarantine, size_t upper_bound)
{ {
while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) { while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) {
void *ptr = quarantine->objs[quarantine->first]; quarantine_obj_t *obj = &quarantine->objs[quarantine->first];
size_t usize = isalloc(ptr, config_prof); assert(obj->usize == isalloc(obj->ptr, config_prof));
idalloc(ptr); idalloc(obj->ptr);
quarantine->curbytes -= usize; quarantine->curbytes -= obj->usize;
quarantine->curobjs--; quarantine->curobjs--;
quarantine->first = (quarantine->first + 1) & ((ZU(1) << quarantine->first = (quarantine->first + 1) & ((ZU(1) <<
quarantine->lg_maxobjs) - 1); quarantine->lg_maxobjs) - 1);
@ -105,10 +118,25 @@ quarantine(void *ptr)
assert(opt_quarantine); assert(opt_quarantine);
quarantine = *quarantine_tsd_get(); quarantine = *quarantine_tsd_get();
if (quarantine == NULL && (quarantine = if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) {
quarantine_init(LG_MAXOBJS_INIT)) == NULL) { if (quarantine == NULL) {
idalloc(ptr); if ((quarantine = quarantine_init(LG_MAXOBJS_INIT)) ==
return; NULL) {
idalloc(ptr);
return;
}
} else {
if (quarantine == QUARANTINE_STATE_PURGATORY) {
/*
* Make a note that quarantine() was called
* after quarantine_cleanup() was called.
*/
quarantine = QUARANTINE_STATE_REINCARNATED;
quarantine_tsd_set(&quarantine);
}
idalloc(ptr);
return;
}
} }
/* /*
* Drain one or more objects if the quarantine size limit would be * Drain one or more objects if the quarantine size limit would be
@ -128,7 +156,9 @@ quarantine(void *ptr)
if (quarantine->curbytes + usize <= opt_quarantine) { if (quarantine->curbytes + usize <= opt_quarantine) {
size_t offset = (quarantine->first + quarantine->curobjs) & size_t offset = (quarantine->first + quarantine->curobjs) &
((ZU(1) << quarantine->lg_maxobjs) - 1); ((ZU(1) << quarantine->lg_maxobjs) - 1);
quarantine->objs[offset] = ptr; quarantine_obj_t *obj = &quarantine->objs[offset];
obj->ptr = ptr;
obj->usize = usize;
quarantine->curbytes += usize; quarantine->curbytes += usize;
quarantine->curobjs++; quarantine->curobjs++;
if (opt_junk) if (opt_junk)
@ -144,9 +174,26 @@ quarantine_cleanup(void *arg)
{ {
quarantine_t *quarantine = *(quarantine_t **)arg; quarantine_t *quarantine = *(quarantine_t **)arg;
if (quarantine != NULL) { if (quarantine == QUARANTINE_STATE_REINCARNATED) {
/*
* Another destructor deallocated memory after this destructor
* was called. Reset quarantine to QUARANTINE_STATE_PURGATORY
* in order to receive another callback.
*/
quarantine = QUARANTINE_STATE_PURGATORY;
quarantine_tsd_set(&quarantine);
} else if (quarantine == QUARANTINE_STATE_PURGATORY) {
/*
* The previous time this destructor was called, we set the key
* to QUARANTINE_STATE_PURGATORY so that other destructors
* wouldn't cause re-creation of the quarantine. This time, do
* nothing, so that the destructor will not be called again.
*/
} else if (quarantine != NULL) {
quarantine_drain(quarantine, 0); quarantine_drain(quarantine, 0);
idalloc(quarantine); idalloc(quarantine);
quarantine = QUARANTINE_STATE_PURGATORY;
quarantine_tsd_set(&quarantine);
} }
} }

30
contrib/jemalloc/src/stats.c
View File

@ -295,16 +295,6 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
abort(); abort();
} }
if (write_cb == NULL) {
/*
* The caller did not provide an alternate write_cb callback
* function, so use the default one. malloc_write() is an
* inline function, so use malloc_message() directly here.
*/
write_cb = je_malloc_message;
cbopaque = NULL;
}
if (opts != NULL) { if (opts != NULL) {
unsigned i; unsigned i;
@ -330,7 +320,8 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
} }
} }
write_cb(cbopaque, "___ Begin jemalloc statistics ___\n"); malloc_cprintf(write_cb, cbopaque,
"___ Begin jemalloc statistics ___\n");
if (general) { if (general) {
int err; int err;
const char *cpv; const char *cpv;
@ -375,7 +366,8 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
" opt."#n": \"%s\"\n", cpv); \ " opt."#n": \"%s\"\n", cpv); \
} }
write_cb(cbopaque, "Run-time option settings:\n"); malloc_cprintf(write_cb, cbopaque,
"Run-time option settings:\n");
OPT_WRITE_BOOL(abort) OPT_WRITE_BOOL(abort)
OPT_WRITE_SIZE_T(lg_chunk) OPT_WRITE_SIZE_T(lg_chunk)
OPT_WRITE_SIZE_T(narenas) OPT_WRITE_SIZE_T(narenas)
@ -425,7 +417,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
"Min active:dirty page ratio per arena: %u:1\n", "Min active:dirty page ratio per arena: %u:1\n",
(1U << ssv)); (1U << ssv));
} else { } else {
write_cb(cbopaque, malloc_cprintf(write_cb, cbopaque,
"Min active:dirty page ratio per arena: N/A\n"); "Min active:dirty page ratio per arena: N/A\n");
} }
if ((err = je_mallctl("arenas.tcache_max", &sv, &ssz, NULL, 0)) if ((err = je_mallctl("arenas.tcache_max", &sv, &ssz, NULL, 0))
@ -447,7 +439,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
" (2^%zd)\n", " (2^%zd)\n",
(((uint64_t)1U) << ssv), ssv); (((uint64_t)1U) << ssv), ssv);
} else { } else {
write_cb(cbopaque, malloc_cprintf(write_cb, cbopaque,
"Average profile dump interval: N/A\n"); "Average profile dump interval: N/A\n");
} }
} }
@ -498,11 +490,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
CTL_GET("arenas.narenas", &narenas, unsigned); CTL_GET("arenas.narenas", &narenas, unsigned);
{ {
bool initialized[narenas]; VARIABLE_ARRAY(bool, initialized, narenas);
size_t isz; size_t isz;
unsigned i, ninitialized; unsigned i, ninitialized;
isz = sizeof(initialized); isz = sizeof(bool) * narenas;
xmallctl("arenas.initialized", initialized, xmallctl("arenas.initialized", initialized,
&isz, NULL, 0); &isz, NULL, 0);
for (i = ninitialized = 0; i < narenas; i++) { for (i = ninitialized = 0; i < narenas; i++) {
@ -527,11 +519,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
CTL_GET("arenas.narenas", &narenas, unsigned); CTL_GET("arenas.narenas", &narenas, unsigned);
{ {
bool initialized[narenas]; VARIABLE_ARRAY(bool, initialized, narenas);
size_t isz; size_t isz;
unsigned i; unsigned i;
isz = sizeof(initialized); isz = sizeof(bool) * narenas;
xmallctl("arenas.initialized", initialized, xmallctl("arenas.initialized", initialized,
&isz, NULL, 0); &isz, NULL, 0);
@ -547,5 +539,5 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque,
} }
} }
} }
write_cb(cbopaque, "--- End jemalloc statistics ---\n"); malloc_cprintf(write_cb, cbopaque, "--- End jemalloc statistics ---\n");
} }

59
contrib/jemalloc/src/tcache.c
View File

@ -24,6 +24,46 @@ size_t tcache_salloc(const void *ptr)
return (arena_salloc(ptr, false)); return (arena_salloc(ptr, false));
} }
void
tcache_event_hard(tcache_t *tcache)
{
size_t binind = tcache->next_gc_bin;
tcache_bin_t *tbin = &tcache->tbins[binind];
tcache_bin_info_t *tbin_info = &tcache_bin_info[binind];
if (tbin->low_water > 0) {
/*
* Flush (ceiling) 3/4 of the objects below the low water mark.
*/
if (binind < NBINS) {
tcache_bin_flush_small(tbin, binind, tbin->ncached -
tbin->low_water + (tbin->low_water >> 2), tcache);
} else {
tcache_bin_flush_large(tbin, binind, tbin->ncached -
tbin->low_water + (tbin->low_water >> 2), tcache);
}
/*
* Reduce fill count by 2X. Limit lg_fill_div such that the
* fill count is always at least 1.
*/
if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1)) >= 1)
tbin->lg_fill_div++;
} else if (tbin->low_water < 0) {
/*
* Increase fill count by 2X. Make sure lg_fill_div stays
* greater than 0.
*/
if (tbin->lg_fill_div > 1)
tbin->lg_fill_div--;
}
tbin->low_water = tbin->ncached;
tcache->next_gc_bin++;
if (tcache->next_gc_bin == nhbins)
tcache->next_gc_bin = 0;
tcache->ev_cnt = 0;
}
void * void *
tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind) tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
{ {
@ -80,12 +120,13 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem,
size_t pageind = ((uintptr_t)ptr - size_t pageind = ((uintptr_t)ptr -
(uintptr_t)chunk) >> LG_PAGE; (uintptr_t)chunk) >> LG_PAGE;
arena_chunk_map_t *mapelm = arena_chunk_map_t *mapelm =
&chunk->map[pageind-map_bias]; arena_mapp_get(chunk, pageind);
if (config_fill && opt_junk) { if (config_fill && opt_junk) {
arena_alloc_junk_small(ptr, arena_alloc_junk_small(ptr,
&arena_bin_info[binind], true); &arena_bin_info[binind], true);
} }
arena_dalloc_bin(arena, chunk, ptr, mapelm); arena_dalloc_bin_locked(arena, chunk, ptr,
mapelm);
} else { } else {
/* /*
* This object was allocated via a different * This object was allocated via a different
@ -158,7 +199,7 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem,
assert(ptr != NULL); assert(ptr != NULL);
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
if (chunk->arena == arena) if (chunk->arena == arena)
arena_dalloc_large(arena, chunk, ptr); arena_dalloc_large_locked(arena, chunk, ptr);
else { else {
/* /*
* This object was allocated via a different * This object was allocated via a different
@ -314,22 +355,14 @@ tcache_destroy(tcache_t *tcache)
arena_t *arena = chunk->arena; arena_t *arena = chunk->arena;
size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >> size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >>
LG_PAGE; LG_PAGE;
arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind);
arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
(uintptr_t)((pageind - (mapelm->bits >> LG_PAGE)) <<
LG_PAGE));
arena_bin_t *bin = run->bin;
malloc_mutex_lock(&bin->lock); arena_dalloc_bin(arena, chunk, tcache, pageind, mapelm);
arena_dalloc_bin(arena, chunk, tcache, mapelm);
malloc_mutex_unlock(&bin->lock);
} else if (tcache_size <= tcache_maxclass) { } else if (tcache_size <= tcache_maxclass) {
arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
arena_t *arena = chunk->arena; arena_t *arena = chunk->arena;
malloc_mutex_lock(&arena->lock);
arena_dalloc_large(arena, chunk, tcache); arena_dalloc_large(arena, chunk, tcache);
malloc_mutex_unlock(&arena->lock);
} else } else
idalloc(tcache); idalloc(tcache);
} }

43
contrib/jemalloc/src/tsd.c
View File

@ -14,7 +14,7 @@ malloc_tsd_malloc(size_t size)
{ {
/* Avoid choose_arena() in order to dodge bootstrapping issues. */ /* Avoid choose_arena() in order to dodge bootstrapping issues. */
return arena_malloc(arenas[0], size, false, false); return (arena_malloc(arenas[0], size, false, false));
} }
void void
@ -31,12 +31,14 @@ malloc_tsd_no_cleanup(void *arg)
not_reached(); not_reached();
} }
#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP #if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32)
JEMALLOC_ATTR(visibility("default")) #ifndef _WIN32
JEMALLOC_EXPORT
#endif
void void
_malloc_thread_cleanup(void) _malloc_thread_cleanup(void)
{ {
bool pending[ncleanups], again; bool pending[MALLOC_TSD_CLEANUPS_MAX], again;
unsigned i; unsigned i;
for (i = 0; i < ncleanups; i++) for (i = 0; i < ncleanups; i++)
@ -70,3 +72,36 @@ malloc_tsd_boot(void)
ncleanups = 0; ncleanups = 0;
} }
#ifdef _WIN32
static BOOL WINAPI
_tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved)
{
switch (fdwReason) {
#ifdef JEMALLOC_LAZY_LOCK
case DLL_THREAD_ATTACH:
isthreaded = true;
break;
#endif
case DLL_THREAD_DETACH:
_malloc_thread_cleanup();
break;
default:
break;
}
return (true);
}
#ifdef _MSC_VER
# ifdef _M_IX86
# pragma comment(linker, "/INCLUDE:__tls_used")
# else
# pragma comment(linker, "/INCLUDE:_tls_used")
# endif
# pragma section(".CRT$XLY",long,read)
#endif
JEMALLOC_SECTION(".CRT$XLY") JEMALLOC_ATTR(used)
static const BOOL (WINAPI *tls_callback)(HINSTANCE hinstDLL,
DWORD fdwReason, LPVOID lpvReserved) = _tls_callback;
#endif

51
contrib/jemalloc/src/util.c
View File

@ -40,8 +40,7 @@ static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s,
/******************************************************************************/ /******************************************************************************/
/* malloc_message() setup. */ /* malloc_message() setup. */
JEMALLOC_CATTR(visibility("hidden"), static) static void
void
wrtmessage(void *cbopaque, const char *s) wrtmessage(void *cbopaque, const char *s)
{ {
@ -57,10 +56,9 @@ wrtmessage(void *cbopaque, const char *s)
#endif #endif
} }
void (*je_malloc_message)(void *, const char *s) JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s);
JEMALLOC_ATTR(visibility("default")) = wrtmessage;
JEMALLOC_CATTR(visibility("hidden"), static) JEMALLOC_ATTR(visibility("hidden"))
void void
wrtmessage_1_0(const char *s1, const char *s2, const char *s3, wrtmessage_1_0(const char *s1, const char *s2, const char *s3,
const char *s4) const char *s4)
@ -76,14 +74,33 @@ void (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3,
const char *s4) = wrtmessage_1_0; const char *s4) = wrtmessage_1_0;
__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0); __sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0);
/*
* Wrapper around malloc_message() that avoids the need for
* je_malloc_message(...) throughout the code.
*/
void
malloc_write(const char *s)
{
if (je_malloc_message != NULL)
je_malloc_message(NULL, s);
else
wrtmessage(NULL, s);
}
/* /*
* glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so
* provide a wrapper. * provide a wrapper.
*/ */
int int
buferror(int errnum, char *buf, size_t buflen) buferror(char *buf, size_t buflen)
{ {
#ifdef _GNU_SOURCE
#ifdef _WIN32
FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(), 0,
(LPSTR)buf, buflen, NULL);
return (0);
#elif defined(_GNU_SOURCE)
char *b = strerror_r(errno, buf, buflen); char *b = strerror_r(errno, buf, buflen);
if (b != buf) { if (b != buf) {
strncpy(buf, b, buflen); strncpy(buf, b, buflen);
@ -104,7 +121,7 @@ malloc_strtoumax(const char *nptr, char **endptr, int base)
const char *p, *ns; const char *p, *ns;
if (base < 0 || base == 1 || base > 36) { if (base < 0 || base == 1 || base > 36) {
errno = EINVAL; set_errno(EINVAL);
return (UINTMAX_MAX); return (UINTMAX_MAX);
} }
b = base; b = base;
@ -179,7 +196,7 @@ malloc_strtoumax(const char *nptr, char **endptr, int base)
ret += digit; ret += digit;
if (ret < pret) { if (ret < pret) {
/* Overflow. */ /* Overflow. */
errno = ERANGE; set_errno(ERANGE);
return (UINTMAX_MAX); return (UINTMAX_MAX);
} }
p++; p++;
@ -299,7 +316,6 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
int ret; int ret;
size_t i; size_t i;
const char *f; const char *f;
va_list tap;
#define APPEND_C(c) do { \ #define APPEND_C(c) do { \
if (i < size) \ if (i < size) \
@ -370,9 +386,6 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
} \ } \
} while (0) } while (0)
if (config_debug)
va_copy(tap, ap);
i = 0; i = 0;
f = format; f = format;
while (true) { while (true) {
@ -431,9 +444,9 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
case '0': case '1': case '2': case '3': case '4': case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9': { case '5': case '6': case '7': case '8': case '9': {
uintmax_t uwidth; uintmax_t uwidth;
errno = 0; set_errno(0);
uwidth = malloc_strtoumax(f, (char **)&f, 10); uwidth = malloc_strtoumax(f, (char **)&f, 10);
assert(uwidth != UINTMAX_MAX || errno != assert(uwidth != UINTMAX_MAX || get_errno() !=
ERANGE); ERANGE);
width = (int)uwidth; width = (int)uwidth;
if (*f == '.') { if (*f == '.') {
@ -457,9 +470,10 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap)
case '0': case '1': case '2': case '3': case '4': case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9': { case '5': case '6': case '7': case '8': case '9': {
uintmax_t uprec; uintmax_t uprec;
errno = 0; set_errno(0);
uprec = malloc_strtoumax(f, (char **)&f, 10); uprec = malloc_strtoumax(f, (char **)&f, 10);
assert(uprec != UINTMAX_MAX || errno != ERANGE); assert(uprec != UINTMAX_MAX || get_errno() !=
ERANGE);
prec = (int)uprec; prec = (int)uprec;
break; break;
} }
@ -610,7 +624,8 @@ malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque,
* function, so use the default one. malloc_write() is an * function, so use the default one. malloc_write() is an
* inline function, so use malloc_message() directly here. * inline function, so use malloc_message() directly here.
*/ */
write_cb = je_malloc_message; write_cb = (je_malloc_message != NULL) ? je_malloc_message :
wrtmessage;
cbopaque = NULL; cbopaque = NULL;
} }