d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
fe81abdeb4
freebsd-nq/contrib/jemalloc/FREEBSD-diffs
Brooks Davis ea69bf7f5d Set LG_VADDR to 48 on RISC-V. 
The Sv48 PTE format is the largest currently defined address space for
RISC-V. It makes no sense to define a larger size and doing so (at
least for 64-bits) forces rtrees down a slow path.

Reviewed by:	vangyzen, jhb, mhorne
Obtained from:	CheriBSD
Sponsored by:	DARPA
Differential Revision:	https://reviews.freebsd.org/D24658
2020-05-04 17:16:30 +00:00

583 lines
17 KiB
Plaintext
Raw Blame History

diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in
index 7fecda7c..d5ca5e86 100644
--- a/doc/jemalloc.xml.in
+++ b/doc/jemalloc.xml.in
@@ -53,11 +53,22 @@
<para>This manual describes jemalloc @jemalloc_version@. More information
can be found at the <ulink
url="http://jemalloc.net/">jemalloc website</ulink>.</para>
+
+ <para>The following configuration options are enabled in libc's built-in
+ jemalloc: <option>--enable-fill</option>,
+ <option>--enable-lazy-lock</option>, <option>--enable-stats</option>,
+ <option>--enable-utrace</option>, <option>--enable-xmalloc</option>, and
+ <option>--with-malloc-conf=abort_conf:false</option>.
+ Additionally, <option>--enable-debug</option> is enabled in development
+ versions of FreeBSD (controlled by the
+ <constant>MALLOC_PRODUCTION</constant> make variable).</para>
+
</refsect1>
<refsynopsisdiv>
<title>SYNOPSIS</title>
<funcsynopsis>
- <funcsynopsisinfo>#include &lt;<filename class="headerfile">jemalloc/jemalloc.h</filename>&gt;</funcsynopsisinfo>
+ <funcsynopsisinfo>#include &lt;<filename class="headerfile">stdlib.h</filename>&gt;
+#include &lt;<filename class="headerfile">malloc_np.h</filename>&gt;</funcsynopsisinfo>
<refsect2>
<title>Standard API</title>
<funcprototype>
@@ -3510,4 +3521,18 @@ malloc_conf = "narenas:1";]]></programlisting></para>
<para>The <function>posix_memalign()</function> function conforms
to IEEE Std 1003.1-2001 (<quote>POSIX.1</quote>).</para>
</refsect1>
+ <refsect1 id="history">
+ <title>HISTORY</title>
+ <para>The <function>malloc_usable_size()</function> and
+ <function>posix_memalign()</function> functions first appeared in FreeBSD
+ 7.0.</para>
+
+ <para>The <function>aligned_alloc()</function>,
+ <function>malloc_stats_print()</function>, and
+ <function>mallctl*()</function> functions first appeared in FreeBSD
+ 10.0.</para>
+
+ <para>The <function>*allocx()</function> functions first appeared in FreeBSD
+ 11.0.</para>
+ </refsect1>
</refentry>
diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h
index 7d6053e2..a0e4f5af 100644
--- a/include/jemalloc/internal/jemalloc_internal_decls.h
+++ b/include/jemalloc/internal/jemalloc_internal_decls.h
@@ -1,6 +1,9 @@
#ifndef JEMALLOC_INTERNAL_DECLS_H
#define JEMALLOC_INTERNAL_DECLS_H
+#include "libc_private.h"
+#include "namespace.h"
+
#include <math.h>
#ifdef _WIN32
# include <windows.h>
diff --git a/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h b/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h
new file mode 100644
index 00000000..0dab1296
--- /dev/null
+++ b/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h
@@ -0,0 +1,9 @@
+#ifndef __clang__
+# undef JEMALLOC_INTERNAL_UNREACHABLE
+# define JEMALLOC_INTERNAL_UNREACHABLE abort
+
+# undef JEMALLOC_C11_ATOMICS
+# undef JEMALLOC_GCC_ATOMIC_ATOMICS
+# undef JEMALLOC_GCC_U8_ATOMIC_ATOMICS
+# undef JEMALLOC_GCC_U8_SYNC_ATOMICS
+#endif
diff --git a/include/jemalloc/internal/jemalloc_preamble.h.in b/include/jemalloc/internal/jemalloc_preamble.h.in
index 3418cbfa..53e30dc4 100644
--- a/include/jemalloc/internal/jemalloc_preamble.h.in
+++ b/include/jemalloc/internal/jemalloc_preamble.h.in
@@ -8,6 +8,9 @@
#include <sys/ktrace.h>
#endif
+#include "un-namespace.h"
+#include "libc_private.h"
+
#define JEMALLOC_NO_DEMANGLE
#ifdef JEMALLOC_JET
# undef JEMALLOC_IS_MALLOC
@@ -79,13 +82,7 @@ static const bool config_fill =
false
#endif
;
-static const bool config_lazy_lock =
-#ifdef JEMALLOC_LAZY_LOCK
- true
-#else
- false
-#endif
- ;
+static const bool config_lazy_lock = true;
static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF;
static const bool config_prof =
#ifdef JEMALLOC_PROF
diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h
index 7c24f072..94af1618 100644
--- a/include/jemalloc/internal/mutex.h
+++ b/include/jemalloc/internal/mutex.h
@@ -135,9 +135,6 @@ struct malloc_mutex_s {
#ifdef JEMALLOC_LAZY_LOCK
extern bool isthreaded;
-#else
-# undef isthreaded /* Undo private_namespace.h definition. */
-# define isthreaded true
#endif
bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
@@ -145,6 +142,7 @@ bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
+bool malloc_mutex_first_thread(void);
bool malloc_mutex_boot(void);
void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex);
diff --git a/include/jemalloc/internal/test_hooks.h b/include/jemalloc/internal/test_hooks.h
index a6351e59..0780c52f 100644
--- a/include/jemalloc/internal/test_hooks.h
+++ b/include/jemalloc/internal/test_hooks.h
@@ -6,13 +6,6 @@ extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)();
#define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn)
-#define open JEMALLOC_HOOK(open, test_hooks_libc_hook)
-#define read JEMALLOC_HOOK(read, test_hooks_libc_hook)
-#define write JEMALLOC_HOOK(write, test_hooks_libc_hook)
-#define readlink JEMALLOC_HOOK(readlink, test_hooks_libc_hook)
-#define close JEMALLOC_HOOK(close, test_hooks_libc_hook)
-#define creat JEMALLOC_HOOK(creat, test_hooks_libc_hook)
-#define secure_getenv JEMALLOC_HOOK(secure_getenv, test_hooks_libc_hook)
/* Note that this is undef'd and re-define'd in src/prof.c. */
#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook)
diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h
index 9ba26004..ecfda5d6 100644
--- a/include/jemalloc/internal/tsd.h
+++ b/include/jemalloc/internal/tsd.h
@@ -198,7 +198,8 @@ struct tsd_s {
t TSD_MANGLE(n);
MALLOC_TSD
#undef O
-};
+/* AddressSanitizer requires TLS data to be aligned to at least 8 bytes. */
+} JEMALLOC_ALIGNED(16);
JEMALLOC_ALWAYS_INLINE uint8_t
tsd_state_get(tsd_t *tsd) {
diff --git a/include/jemalloc/jemalloc_FreeBSD.h b/include/jemalloc/jemalloc_FreeBSD.h
new file mode 100644
index 00000000..b752b0e7
--- /dev/null
+++ b/include/jemalloc/jemalloc_FreeBSD.h
@@ -0,0 +1,185 @@
+/*
+ * Override settings that were generated in jemalloc_defs.h as necessary.
+ */
+
+#undef JEMALLOC_OVERRIDE_VALLOC
+
+#ifndef MALLOC_PRODUCTION
+#define JEMALLOC_DEBUG
+#endif
+
+#undef JEMALLOC_DSS
+
+#undef JEMALLOC_BACKGROUND_THREAD
+
+/*
+ * The following are architecture-dependent, so conditionally define them for
+ * each supported architecture.
+ */
+#undef JEMALLOC_TLS_MODEL
+#undef LG_PAGE
+#undef LG_VADDR
+#undef LG_SIZEOF_PTR
+#undef LG_SIZEOF_INT
+#undef LG_SIZEOF_LONG
+#undef LG_SIZEOF_INTMAX_T
+
+#ifdef __i386__
+# define LG_VADDR 32
+# define LG_SIZEOF_PTR 2
+# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
+#endif
+#ifdef __ia64__
+# define LG_VADDR 64
+# define LG_SIZEOF_PTR 3
+#endif
+#ifdef __sparc64__
+# define LG_VADDR 64
+# define LG_SIZEOF_PTR 3
+# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
+#endif
+#ifdef __amd64__
+# define LG_VADDR 48
+# define LG_SIZEOF_PTR 3
+# define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
+#endif
+#ifdef __arm__
+# define LG_VADDR 32
+# define LG_SIZEOF_PTR 2
+#endif
+#ifdef __aarch64__
+# define LG_VADDR 48
+# define LG_SIZEOF_PTR 3
+#endif
+#ifdef __mips__
+#ifdef __mips_n64
+# define LG_VADDR 64
+# define LG_SIZEOF_PTR 3
+#else
+# define LG_VADDR 32
+# define LG_SIZEOF_PTR 2
+#endif
+#endif
+#ifdef __powerpc64__
+# define LG_VADDR 64
+# define LG_SIZEOF_PTR 3
+#elif defined(__powerpc__)
+# define LG_VADDR 32
+# define LG_SIZEOF_PTR 2
+#endif
+#ifdef __riscv
+# define LG_VADDR 48
+# define LG_SIZEOF_PTR 3
+#endif
+
+#ifndef JEMALLOC_TLS_MODEL
+# define JEMALLOC_TLS_MODEL /* Default. */
+#endif
+
+#define LG_PAGE PAGE_SHIFT
+#define LG_SIZEOF_INT 2
+#define LG_SIZEOF_LONG LG_SIZEOF_PTR
+#define LG_SIZEOF_INTMAX_T 3
+
+#undef CPU_SPINWAIT
+#include <machine/cpu.h>
+#include <machine/cpufunc.h>
+#define CPU_SPINWAIT cpu_spinwait()
+
+/* Disable lazy-lock machinery, mangle isthreaded, and adjust its type. */
+#undef JEMALLOC_LAZY_LOCK
+extern int __isthreaded;
+#define isthreaded ((bool)__isthreaded)
+
+/* Mangle. */
+#undef je_malloc
+#undef je_calloc
+#undef je_posix_memalign
+#undef je_aligned_alloc
+#undef je_realloc
+#undef je_free
+#undef je_malloc_usable_size
+#undef je_mallocx
+#undef je_rallocx
+#undef je_xallocx
+#undef je_sallocx
+#undef je_dallocx
+#undef je_sdallocx
+#undef je_nallocx
+#undef je_mallctl
+#undef je_mallctlnametomib
+#undef je_mallctlbymib
+#undef je_malloc_stats_print
+#undef je_allocm
+#undef je_rallocm
+#undef je_sallocm
+#undef je_dallocm
+#undef je_nallocm
+#define je_malloc __malloc
+#define je_calloc __calloc
+#define je_posix_memalign __posix_memalign
+#define je_aligned_alloc __aligned_alloc
+#define je_realloc __realloc
+#define je_free __free
+#define je_malloc_usable_size __malloc_usable_size
+#define je_mallocx __mallocx
+#define je_rallocx __rallocx
+#define je_xallocx __xallocx
+#define je_sallocx __sallocx
+#define je_dallocx __dallocx
+#define je_sdallocx __sdallocx
+#define je_nallocx __nallocx
+#define je_mallctl __mallctl
+#define je_mallctlnametomib __mallctlnametomib
+#define je_mallctlbymib __mallctlbymib
+#define je_malloc_stats_print __malloc_stats_print
+#define je_allocm __allocm
+#define je_rallocm __rallocm
+#define je_sallocm __sallocm
+#define je_dallocm __dallocm
+#define je_nallocm __nallocm
+#define open _open
+#define read _read
+#define write _write
+#define close _close
+#define pthread_join _pthread_join
+#define pthread_once _pthread_once
+#define pthread_self _pthread_self
+#define pthread_equal _pthread_equal
+#define pthread_mutex_lock _pthread_mutex_lock
+#define pthread_mutex_trylock _pthread_mutex_trylock
+#define pthread_mutex_unlock _pthread_mutex_unlock
+#define pthread_cond_init _pthread_cond_init
+#define pthread_cond_wait _pthread_cond_wait
+#define pthread_cond_timedwait _pthread_cond_timedwait
+#define pthread_cond_signal _pthread_cond_signal
+
+#ifdef JEMALLOC_C_
+/*
+ * Define 'weak' symbols so that an application can have its own versions
+ * of malloc, calloc, realloc, free, et al.
+ */
+__weak_reference(__malloc, malloc);
+__weak_reference(__calloc, calloc);
+__weak_reference(__posix_memalign, posix_memalign);
+__weak_reference(__aligned_alloc, aligned_alloc);
+__weak_reference(__realloc, realloc);
+__weak_reference(__free, free);
+__weak_reference(__malloc_usable_size, malloc_usable_size);
+__weak_reference(__mallocx, mallocx);
+__weak_reference(__rallocx, rallocx);
+__weak_reference(__xallocx, xallocx);
+__weak_reference(__sallocx, sallocx);
+__weak_reference(__dallocx, dallocx);
+__weak_reference(__sdallocx, sdallocx);
+__weak_reference(__nallocx, nallocx);
+__weak_reference(__mallctl, mallctl);
+__weak_reference(__mallctlnametomib, mallctlnametomib);
+__weak_reference(__mallctlbymib, mallctlbymib);
+__weak_reference(__malloc_stats_print, malloc_stats_print);
+__weak_reference(__allocm, allocm);
+__weak_reference(__rallocm, rallocm);
+__weak_reference(__sallocm, sallocm);
+__weak_reference(__dallocm, dallocm);
+__weak_reference(__nallocm, nallocm);
+#endif
diff --git a/include/jemalloc/jemalloc_rename.sh b/include/jemalloc/jemalloc_rename.sh
index f9438912..47d032c1 100755
--- a/include/jemalloc/jemalloc_rename.sh
+++ b/include/jemalloc/jemalloc_rename.sh
@@ -19,4 +19,6 @@ done
cat <<EOF
#endif
+
+#include "jemalloc_FreeBSD.h"
EOF
diff --git a/src/jemalloc.c b/src/jemalloc.c
index ed13718d..fefb719a 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -23,6 +23,10 @@
/******************************************************************************/
/* Data. */
+/* Work around <http://llvm.org/bugs/show_bug.cgi?id=12623>: */
+const char *__malloc_options_1_0 = NULL;
+__sym_compat(_malloc_options, __malloc_options_1_0, FBSD_1.0);
+
/* Runtime configuration options. */
const char *je_malloc_conf
#ifndef _WIN32
@@ -2660,25 +2664,6 @@ je_realloc(void *ptr, size_t arg_size) {
LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size);
if (unlikely(size == 0)) {
- if (ptr != NULL) {
- /* realloc(ptr, 0) is equivalent to free(ptr). */
- UTRACE(ptr, 0, 0);
- tcache_t *tcache;
- tsd_t *tsd = tsd_fetch();
- if (tsd_reentrancy_level_get(tsd) == 0) {
- tcache = tcache_get(tsd);
- } else {
- tcache = NULL;
- }
-
- uintptr_t args[3] = {(uintptr_t)ptr, size};
- hook_invoke_dalloc(hook_dalloc_realloc, ptr, args);
-
- ifree(tsd, ptr, tcache, true);
-
- LOG("core.realloc.exit", "result: %p", NULL);
- return NULL;
- }
size = 1;
}
@@ -3750,6 +3735,103 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
* End non-standard functions.
*/
/******************************************************************************/
+/*
+ * Begin compatibility functions.
+ */
+
+#define ALLOCM_LG_ALIGN(la) (la)
+#define ALLOCM_ALIGN(a) (ffsl(a)-1)
+#define ALLOCM_ZERO ((int)0x40)
+#define ALLOCM_NO_MOVE ((int)0x80)
+
+#define ALLOCM_SUCCESS 0
+#define ALLOCM_ERR_OOM 1
+#define ALLOCM_ERR_NOT_MOVED 2
+
+int
+je_allocm(void **ptr, size_t *rsize, size_t size, int flags) {
+ assert(ptr != NULL);
+
+ void *p = je_mallocx(size, flags);
+ if (p == NULL) {
+ return (ALLOCM_ERR_OOM);
+ }
+ if (rsize != NULL) {
+ *rsize = isalloc(tsdn_fetch(), p);
+ }
+ *ptr = p;
+ return ALLOCM_SUCCESS;
+}
+
+int
+je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) {
+ assert(ptr != NULL);
+ assert(*ptr != NULL);
+ assert(size != 0);
+ assert(SIZE_T_MAX - size >= extra);
+
+ int ret;
+ bool no_move = flags & ALLOCM_NO_MOVE;
+
+ if (no_move) {
+ size_t usize = je_xallocx(*ptr, size, extra, flags);
+ ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED;
+ if (rsize != NULL) {
+ *rsize = usize;
+ }
+ } else {
+ void *p = je_rallocx(*ptr, size+extra, flags);
+ if (p != NULL) {
+ *ptr = p;
+ ret = ALLOCM_SUCCESS;
+ } else {
+ ret = ALLOCM_ERR_OOM;
+ }
+ if (rsize != NULL) {
+ *rsize = isalloc(tsdn_fetch(), *ptr);
+ }
+ }
+ return ret;
+}
+
+int
+je_sallocm(const void *ptr, size_t *rsize, int flags) {
+ assert(rsize != NULL);
+ *rsize = je_sallocx(ptr, flags);
+ return ALLOCM_SUCCESS;
+}
+
+int
+je_dallocm(void *ptr, int flags) {
+ je_dallocx(ptr, flags);
+ return ALLOCM_SUCCESS;
+}
+
+int
+je_nallocm(size_t *rsize, size_t size, int flags) {
+ size_t usize = je_nallocx(size, flags);
+ if (usize == 0) {
+ return ALLOCM_ERR_OOM;
+ }
+ if (rsize != NULL) {
+ *rsize = usize;
+ }
+ return ALLOCM_SUCCESS;
+}
+
+#undef ALLOCM_LG_ALIGN
+#undef ALLOCM_ALIGN
+#undef ALLOCM_ZERO
+#undef ALLOCM_NO_MOVE
+
+#undef ALLOCM_SUCCESS
+#undef ALLOCM_ERR_OOM
+#undef ALLOCM_ERR_NOT_MOVED
+
+/*
+ * End compatibility functions.
+ */
+/******************************************************************************/
/*
* The following functions are used by threading libraries for protection of
* malloc during fork().
@@ -3919,4 +4001,11 @@ jemalloc_postfork_child(void) {
ctl_postfork_child(tsd_tsdn(tsd));
}
+void
+_malloc_first_thread(void)
+{
+
+ (void)malloc_mutex_first_thread();
+}
+
/******************************************************************************/
diff --git a/src/malloc_io.c b/src/malloc_io.c
index d7cb0f52..cda589c4 100644
--- a/src/malloc_io.c
+++ b/src/malloc_io.c
@@ -75,6 +75,20 @@ wrtmessage(void *cbopaque, const char *s) {
JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s);
+JEMALLOC_ATTR(visibility("hidden"))
+void
+wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) {
+
+ wrtmessage(NULL, s1);
+ wrtmessage(NULL, s2);
+ wrtmessage(NULL, s3);
+ wrtmessage(NULL, s4);
+}
+
+void (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3,
+ const char *s4) = wrtmessage_1_0;
+__sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0);
+
/*
* Wrapper around malloc_message() that avoids the need for
* je_malloc_message(...) throughout the code.
diff --git a/src/mutex.c b/src/mutex.c
index 3f920f5b..88a7730c 100644
--- a/src/mutex.c
+++ b/src/mutex.c
@@ -41,6 +41,17 @@ pthread_create(pthread_t *__restrict thread,
#ifdef JEMALLOC_MUTEX_INIT_CB
JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
void *(calloc_cb)(size_t, size_t));
+
+#pragma weak _pthread_mutex_init_calloc_cb
+int
+_pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
+ void *(calloc_cb)(size_t, size_t))
+{
+
+ return (((int (*)(pthread_mutex_t *, void *(*)(size_t, size_t)))
+ __libc_interposing[INTERPOS__pthread_mutex_init_calloc_cb])(mutex,
+ calloc_cb));
+}
#endif
void
@@ -131,6 +142,16 @@ mutex_addr_comp(const witness_t *witness1, void *mutex1,
}
}
+bool
+malloc_mutex_first_thread(void) {
+
+#ifndef JEMALLOC_MUTEX_INIT_CB
+ return (malloc_mutex_first_thread());
+#else
+ return (false);
+#endif
+}
+
bool
malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
witness_rank_t rank, malloc_mutex_lock_order_t lock_order) {
Reference in New Issue View Git Blame Copy Permalink