diff --git a/lib/libthr/Makefile b/lib/libthr/Makefile index 9b932cdb7aac..56905d36e8c5 100644 --- a/lib/libthr/Makefile +++ b/lib/libthr/Makefile @@ -27,6 +27,7 @@ CFLAGS+=-I${SRCTOP}/lib/libthread_db CFLAGS+=-Winline CFLAGS.thr_stack.c+= -Wno-cast-align +CFLAGS.malloc.c+= -Wno-cast-align .include .if !(${COMPILER_TYPE} == "gcc" && ${COMPILER_VERSION} < 40300) CFLAGS.thr_symbols.c+= -Wno-missing-variable-declarations @@ -50,12 +51,14 @@ CFLAGS+=-D_PTHREADS_INVARIANTS PRECIOUSLIB= .PATH: ${.CURDIR}/arch/${MACHINE_CPUARCH}/${MACHINE_CPUARCH} +.PATH: ${SRCTOP}/libexec/rtld-elf .if exists(${.CURDIR}/arch/${MACHINE_CPUARCH}/Makefile.inc) .include "${.CURDIR}/arch/${MACHINE_CPUARCH}/Makefile.inc" .endif .include "${.CURDIR}/sys/Makefile.inc" .include "${.CURDIR}/thread/Makefile.inc" +SRCS+= malloc.c .if ${MK_INSTALLLIB} != "no" SYMLINKS+=lib${LIB}.a ${LIBDIR}/libpthread.a diff --git a/lib/libthr/thread/Makefile.inc b/lib/libthr/thread/Makefile.inc index 795ed3989abe..34b881a69637 100644 --- a/lib/libthr/thread/Makefile.inc +++ b/lib/libthr/thread/Makefile.inc @@ -31,6 +31,7 @@ SRCS+= \ thr_kern.c \ thr_kill.c \ thr_main_np.c \ + thr_malloc.c \ thr_multi_np.c \ thr_mutex.c \ thr_mutexattr.c \ diff --git a/lib/libthr/thread/thr_fork.c b/lib/libthr/thread/thr_fork.c index 8d969cbf9718..2e151618ffbc 100644 --- a/lib/libthr/thread/thr_fork.c +++ b/lib/libthr/thread/thr_fork.c @@ -170,6 +170,7 @@ __thr_fork(void) */ if (_thr_isthreaded() != 0) { was_threaded = 1; + __thr_malloc_prefork(curthread); _malloc_prefork(); __thr_pshared_atfork_pre(); _rtld_atfork_pre(rtld_locks); @@ -197,6 +198,10 @@ __thr_fork(void) */ curthread->tlflags &= ~TLFLAGS_IN_TDLIST; + /* before thr_self() */ + if (was_threaded) + __thr_malloc_postfork(curthread); + /* child is a new kernel thread. */ thr_self(&curthread->tid); @@ -241,6 +246,7 @@ __thr_fork(void) _thr_signal_postfork(); if (was_threaded) { + __thr_malloc_postfork(curthread); _rtld_atfork_post(rtld_locks); __thr_pshared_atfork_post(); _malloc_postfork(); diff --git a/lib/libthr/thread/thr_init.c b/lib/libthr/thread/thr_init.c index 4781f8956c48..1568529e8c51 100644 --- a/lib/libthr/thread/thr_init.c +++ b/lib/libthr/thread/thr_init.c @@ -461,6 +461,7 @@ init_private(void) */ if (init_once == 0) { __thr_pshared_init(); + __thr_malloc_init(); /* Find the stack top */ mib[0] = CTL_KERN; mib[1] = KERN_USRSTACK; diff --git a/lib/libthr/thread/thr_malloc.c b/lib/libthr/thread/thr_malloc.c new file mode 100644 index 000000000000..157c72f10d6c --- /dev/null +++ b/lib/libthr/thread/thr_malloc.c @@ -0,0 +1,137 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * + * Copyright (c) 2019 The FreeBSD Foundation + * All rights reserved. + * + * This software was developed by Konstantin Belousov + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include "thr_private.h" + +int npagesizes; +size_t *pagesizes; +static size_t pagesizes_d[2]; +static struct umutex thr_malloc_umtx; + +void +__thr_malloc_init(void) +{ + + npagesizes = getpagesizes(pagesizes_d, nitems(pagesizes_d)); + if (npagesizes == -1) { + npagesizes = 1; + pagesizes_d[0] = PAGE_SIZE; + } + pagesizes = pagesizes_d; + _thr_umutex_init(&thr_malloc_umtx); +} + +static void +thr_malloc_lock(struct pthread *curthread) +{ + + curthread->locklevel++; + _thr_umutex_lock(&thr_malloc_umtx, TID(curthread)); +} + +static void +thr_malloc_unlock(struct pthread *curthread) +{ + + _thr_umutex_unlock(&thr_malloc_umtx, TID(curthread)); + curthread->locklevel--; + _thr_ast(curthread); +} + +void * +__thr_calloc(size_t num, size_t size) +{ + struct pthread *curthread; + void *res; + + curthread = _get_curthread(); + thr_malloc_lock(curthread); + res = __crt_calloc(num, size); + thr_malloc_unlock(curthread); + return (res); +} + +void +__thr_free(void *cp) +{ + struct pthread *curthread; + + curthread = _get_curthread(); + thr_malloc_lock(curthread); + __crt_free(cp); + thr_malloc_unlock(curthread); +} + +void * +__thr_malloc(size_t nbytes) +{ + struct pthread *curthread; + void *res; + + curthread = _get_curthread(); + thr_malloc_lock(curthread); + res = __crt_malloc(nbytes); + thr_malloc_unlock(curthread); + return (res); +} + +void * +__thr_realloc(void *cp, size_t nbytes) +{ + struct pthread *curthread; + void *res; + + curthread = _get_curthread(); + thr_malloc_lock(curthread); + res = __crt_realloc(cp, nbytes); + thr_malloc_unlock(curthread); + return (res); +} + +void +__thr_malloc_prefork(struct pthread *curthread) +{ + + _thr_umutex_lock(&thr_malloc_umtx, TID(curthread)); +} + +void +__thr_malloc_postfork(struct pthread *curthread) +{ + + _thr_umutex_unlock(&thr_malloc_umtx, TID(curthread)); +} diff --git a/lib/libthr/thread/thr_mutex.c b/lib/libthr/thread/thr_mutex.c index d2d9f5b54c10..f6f37c1264e2 100644 --- a/lib/libthr/thread/thr_mutex.c +++ b/lib/libthr/thread/thr_mutex.c @@ -306,10 +306,11 @@ init_static(struct pthread *thread, pthread_mutex_t *mutex) THR_LOCK_ACQUIRE(thread, &_mutex_static_lock); if (*mutex == THR_MUTEX_INITIALIZER) - ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc); + ret = mutex_init(mutex, &_pthread_mutexattr_default, + __thr_calloc); else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER) ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default, - calloc); + __thr_calloc); else ret = 0; THR_LOCK_RELEASE(thread, &_mutex_static_lock); @@ -390,7 +391,7 @@ __pthread_mutex_init(pthread_mutex_t * __restrict mutex, if (mutex_attr == NULL || (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) { return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL, - calloc)); + __thr_calloc)); } pmtx = __thr_pshared_offpage(__DECONST(void *, mutex), 1); if (pmtx == NULL) @@ -483,7 +484,7 @@ _pthread_mutex_destroy(pthread_mutex_t *mutex) } else { *mutex = THR_MUTEX_DESTROYED; mutex_assert_not_owned(_get_curthread(), m); - free(m); + __thr_free(m); ret = 0; } } diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h index b2816c03b44b..897784ad721e 100644 --- a/lib/libthr/thread/thr_private.h +++ b/lib/libthr/thread/thr_private.h @@ -1003,6 +1003,14 @@ void __thr_pshared_destroy(void *key) __hidden; void __thr_pshared_atfork_pre(void) __hidden; void __thr_pshared_atfork_post(void) __hidden; +void *__thr_calloc(size_t num, size_t size); +void __thr_free(void *cp); +void *__thr_malloc(size_t nbytes); +void *__thr_realloc(void *cp, size_t nbytes); +void __thr_malloc_init(void); +void __thr_malloc_prefork(struct pthread *curthread); +void __thr_malloc_postfork(struct pthread *curthread); + __END_DECLS __NULLABILITY_PRAGMA_POP diff --git a/lib/libthr/thread/thr_spec.c b/lib/libthr/thread/thr_spec.c index faa88e2fc8ba..d4da21d096a9 100644 --- a/lib/libthr/thread/thr_spec.c +++ b/lib/libthr/thread/thr_spec.c @@ -155,8 +155,7 @@ _thread_cleanupspecific(void) } } THR_LOCK_RELEASE(curthread, &_keytable_lock); - munmap(curthread->specific, PTHREAD_KEYS_MAX * sizeof(struct - pthread_specific_elem)); + __thr_free(curthread->specific); curthread->specific = NULL; if (curthread->specific_data_count > 0) { stderr_debug("Thread %p has exited with leftover " @@ -179,10 +178,9 @@ _pthread_setspecific(pthread_key_t userkey, const void *value) pthread = _get_curthread(); if (pthread->specific == NULL) { - tmp = mmap(NULL, PTHREAD_KEYS_MAX * - sizeof(struct pthread_specific_elem), - PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); - if (tmp == MAP_FAILED) + tmp = __thr_calloc(PTHREAD_KEYS_MAX, + sizeof(struct pthread_specific_elem)); + if (tmp == NULL) return (ENOMEM); pthread->specific = tmp; }