Add __cxa_thread_atexit(3) API implementation.

This is the backing feature to implement C++11 thread storage duration
specified by the thread_local keyword.  A destructor for given
thread-local object is registered to be executed at the thread
termination time using __cxa_thread_atexit().  Libc calls the
__cxa_thread_calls_dtors() during exit(3), before finalizers and
atexit functions, and libthr calls the function at the thread
termination time, after the stack unwinding and thread-specific key
destruction.

There are several uncertainties in the API which lacks a formal
specification.  Among them:
- is it allowed to register destructors during destructing;
	we allow, but limiting the nesting level.  If too many iterations
	detected, a diagnostic is issued to stderr and thread forcibly
	terminates for now.
- how to handle destructors which belong to an unloading dso;
	for now, we ignore destructor calls for such entries, and
	issue a diagnostic.  Linux does prevent dso unload until all
	threads with destructors from the dso terminated.
It is supposed that the diagnostics allow to detect real-world
applications relying on the above details and possibly adjust
our implementation.  Right now the choices were to provide the slim
API (but that rarely stands the practice test).

Tests are added to check generic functionality and to specify some of
the above implementation choices.

Submitted by:	Mahdi Mokhtari <mokhi64_gmail.com>
Reviewed by:	theraven
Discussed with:	dim (detection of -std=c++11 supoort for tests)
Sponsored by:	The FreeBSD Foundation (my involvement)
MFC after:	2 weeks
Differential revisions:	https://reviews.freebsd.org/D7224,
    https://reviews.freebsd.org/D7427
This commit is contained in:
Konstantin Belousov 2016-08-06 13:32:40 +00:00
parent c3bbb3be4e
commit b585cd3e2c
10 changed files with 461 additions and 2 deletions

View File

@ -266,6 +266,12 @@ extern const char *__progname;
*/
void _malloc_thread_cleanup(void);
/*
* This function is used by the threading libraries to notify libc that a
* thread is exiting, so its thread-local dtors should be called.
*/
void __cxa_thread_call_dtors(void);
/*
* These functions are used by the threading libraries in order to protect
* malloc across fork().

View File

@ -5,7 +5,7 @@
.PATH: ${LIBC_SRCTOP}/${LIBC_ARCH}/stdlib ${LIBC_SRCTOP}/stdlib
MISRCS+=_Exit.c a64l.c abort.c abs.c atexit.c atof.c atoi.c atol.c atoll.c \
bsearch.c div.c exit.c getenv.c getopt.c getopt_long.c \
bsearch.c cxa_thread_atexit.c div.c exit.c getenv.c getopt.c getopt_long.c \
getsubopt.c hcreate.c hcreate_r.c hdestroy_r.c heapsort.c heapsort_b.c \
hsearch_r.c imaxabs.c imaxdiv.c \
insque.c l64a.c labs.c ldiv.c llabs.c lldiv.c lsearch.c \

View File

@ -116,8 +116,13 @@ FBSD_1.4 {
reallocarray;
};
FBSD_1.5 {
__cxa_thread_atexit;
};
FBSDprivate_1.0 {
__system;
_system;
__libc_system;
__cxa_thread_call_dtors;
};

View File

@ -0,0 +1,140 @@
/*-
* Copyright (c) 2016 Mahdi Mokhtari <mokhi64@gmail.com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/queue.h>
#include "namespace.h"
#include <errno.h>
#include <link.h>
#include <pthread.h>
#include <stddef.h>
#include <stdlib.h>
#include <stdio.h>
#include "un-namespace.h"
#include "libc_private.h"
/*
* C++11 introduces the thread_local scope (like __thread with some
* additions). As a key-feature it should support non-trivial
* destructors, registered with __cxa_thread_atexit() to be executed
* at the thread termination.
*
* The implemention keeps a _Thread_local list of destructors per each
* thread, and calls __cxa_thread_call_dtors() on each thread's exit
* to do cleanup. For a thread calling exit(3), in particular, for
* the initial thread returning from main(), we call
* __cxa_thread_call_dtors() inside exit().
*
* It could be possible that a dynamically loaded library, use
* thread_local variable but is dlclose()'d before thread exit. The
* destructor of this variable will then try to access the address,
* for calling it but it's unloaded, so it'll crash. We're using
* __elf_phdr_match_addr() to detect and prevent such cases and so
* prevent the crash.
*/
#define CXA_DTORS_ITERATIONS 4
struct cxa_thread_dtor {
void *obj;
void (*func)(void *);
void *dso;
LIST_ENTRY(cxa_thread_dtor) entry;
};
static _Thread_local LIST_HEAD(dtor_list, cxa_thread_dtor) dtors =
LIST_HEAD_INITIALIZER(dtors);
int
__cxa_thread_atexit(void (*dtor_func)(void *), void *obj, void *dso_symbol)
{
struct cxa_thread_dtor *new_dtor;
new_dtor = malloc(sizeof(*new_dtor));
if (new_dtor == NULL) {
errno = ENOMEM; /* forcibly override malloc(3) error */
return (-1);
}
new_dtor->obj = obj;
new_dtor->func = dtor_func;
new_dtor->dso = dso_symbol;
LIST_INSERT_HEAD(&dtors, new_dtor, entry);
return (0);
}
static void
walk_cb_call(struct cxa_thread_dtor *dtor)
{
struct dl_phdr_info phdr_info;
if (_rtld_addr_phdr(dtor->dso, &phdr_info) &&
__elf_phdr_match_addr(&phdr_info, dtor->func))
dtor->func(dtor->obj);
else
fprintf(stderr, "__cxa_thread_call_dtors: dtr %p from "
"unloaded dso, skipping\n", (void *)(dtor->func));
}
static void
walk_cb_nocall(struct cxa_thread_dtor *dtor __unused)
{
}
static void
cxa_thread_walk(void (*cb)(struct cxa_thread_dtor *))
{
struct cxa_thread_dtor *dtor, *tdtor;
LIST_FOREACH_SAFE(dtor, &dtors, entry, tdtor) {
LIST_REMOVE(dtor, entry);
cb(dtor);
free(dtor);
}
}
/*
* This is the callback function we use to call destructors, once for
* each thread. It is called in exit(3) in libc/stdlib/exit.c and
* before exit_thread() in libthr/thread/thr_exit.c.
*/
void
__cxa_thread_call_dtors(void)
{
int i;
for (i = 0; i < CXA_DTORS_ITERATIONS && !LIST_EMPTY(&dtors); i++)
cxa_thread_walk(walk_cb_call);
if (!LIST_EMPTY(&dtors)) {
fprintf(stderr, "Thread %p is exiting with more "
"thread-specific dtors created after %d iterations "
"of destructor calls\n",
_pthread_self(), i);
cxa_thread_walk(walk_cb_nocall);
}
}

View File

@ -63,6 +63,12 @@ exit(int status)
_thread_autoinit_dummy_decl = 1;
/*
* We're dealing with cleaning up thread_local destructors in the case of
* the process termination through main() exit.
* Other cases are handled elsewhere.
*/
__cxa_thread_call_dtors();
__cxa_finalize(NULL);
if (__cleanup)
(*__cleanup)();

View File

@ -1,9 +1,15 @@
# $FreeBSD$
.include <src.opts.mk>
ATF_TESTS_C+= heapsort_test
ATF_TESTS_C+= mergesort_test
ATF_TESTS_C+= qsort_test
ATF_TESTS_C+= tsearch_test
.if ${COMPILER_FEATURES:Mc++11}
ATF_TESTS_CXX+= cxa_thread_atexit_test
ATF_TESTS_CXX+= cxa_thread_atexit_nothr_test
.endif
# TODO: t_getenv_thread, t_mi_vector_hash
NETBSD_ATF_TESTS_C+= abs_test
@ -33,6 +39,10 @@ PROGS+= h_getopt h_getopt_long
CFLAGS+= -I${.CURDIR}
CXXFLAGS.cxa_thread_atexit_test+= -std=c++11
CXXFLAGS.cxa_thread_atexit_nothr_test+= -std=c++11
LIBADD.cxa_thread_atexit_test+= pthread
.for t in h_getopt h_getopt_long
CFLAGS.$t+= -I${LIBNETBSD_SRCDIR} -I${SRCTOP}/contrib/netbsd-tests
LDFLAGS.$t+= -L${LIBNETBSD_OBJDIR}

View File

@ -8,7 +8,10 @@ DIRDEPS = \
include/xlocale \
lib/${CSU_DIR} \
lib/atf/libatf-c \
lib/atf/libatf-c++ \
lib/libc \
lib/libc++ \
lib/libthr \
lib/libcompiler_rt \
lib/libnetbsd \
lib/libutil \

View File

@ -0,0 +1,102 @@
/*-
* Copyright (c) 2016 Mahdi Mokhtari <mokhi64@gmail.com>
* Copyright (c) 2016 The FreeBSD Foundation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <dlfcn.h>
#include <atf-c++.hpp>
#include <cstdio>
#include <cstdlib>
static FILE *output = NULL;
struct Foo {
Foo() { ATF_REQUIRE(fprintf(output, "Created\n") > 0); }
~Foo() { ATF_REQUIRE(fprintf(output, "Destroyed\n") > 0); }
void use() { ATF_REQUIRE(fprintf(output, "Used\n") > 0); }
};
static thread_local Foo f;
/*
* This test must not be linked to libpthread.
*/
ATF_TEST_CASE_WITHOUT_HEAD(cxx__nothr);
ATF_TEST_CASE_BODY(cxx__nothr)
{
void *libthr_handle;
/* Avoid coredump during f construction. */
output = stderr;
libthr_handle = dlopen("libthr.so.3", RTLD_LAZY | RTLD_GLOBAL |
RTLD_NOLOAD);
ATF_REQUIRE(libthr_handle == NULL);
}
static void
check_local_main(void)
{
static const char out_log[] = "Created\nUsed\nDestroyed\n";
fflush(output);
ATF_REQUIRE(atf::utils::compare_file("test_main.txt", out_log));
}
ATF_TEST_CASE_WITHOUT_HEAD(cxx__thread_local_main);
ATF_TEST_CASE_BODY(cxx__thread_local_main)
{
ATF_REQUIRE((output = fopen("test_main.txt", "w")) != NULL);
f.use();
atexit(check_local_main);
}
extern "C" int __cxa_thread_atexit(void (*)(void *), void *, void *);
static void
again(void *arg)
{
__cxa_thread_atexit(again, arg, &output);
}
ATF_TEST_CASE_WITHOUT_HEAD(cxx__thread_inf_dtors);
ATF_TEST_CASE_BODY(cxx__thread_inf_dtors)
{
again(NULL);
}
ATF_INIT_TEST_CASES(tcs)
{
ATF_ADD_TEST_CASE(tcs, cxx__nothr);
ATF_ADD_TEST_CASE(tcs, cxx__thread_local_main);
ATF_ADD_TEST_CASE(tcs, cxx__thread_inf_dtors);
}

View File

@ -0,0 +1,180 @@
/*-
* Copyright (c) 2016 Mahdi Mokhtari <mokhi64@gmail.com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <dlfcn.h>
#include <atf-c++.hpp>
#include <cstdio>
#include <cstdlib>
#include <thread>
static FILE *output = NULL;
struct Foo {
Foo() { ATF_REQUIRE(fprintf(output, "Created\n") > 0); }
~Foo() { ATF_REQUIRE(fprintf(output, "Destroyed\n") > 0); }
void use() { ATF_REQUIRE(fprintf(output, "Used\n") > 0); }
};
struct Bar {
Bar() {}
~Bar() {
thread_local static Foo foo;
ATF_REQUIRE(fprintf(output, "DIED\n") > 0);
}
void use() {}
};
extern "C" int __cxa_thread_atexit(void (*)(void *), void *, void *);
static void
again(void *arg)
{
__cxa_thread_atexit(again, arg, &output);
}
struct Baz {
Baz() {}
~Baz() {
again(NULL);
}
void use() {}
};
static thread_local Foo f;
static thread_local Foo g;
static thread_local Bar h;
static thread_local Baz e;
/*
* This test must be linked to libpthread.
*/
ATF_TEST_CASE_WITHOUT_HEAD(cxx__thr);
ATF_TEST_CASE_BODY(cxx__thr)
{
void *libthr_handle;
/* Avoid coredump during f construction. */
output = stderr;
libthr_handle = dlopen("libthr.so.3", RTLD_LAZY | RTLD_GLOBAL |
RTLD_NOLOAD);
ATF_REQUIRE(libthr_handle != NULL);
dlclose(libthr_handle);
}
/*
* In this test f.use() will test cxa_thread_atexit() in non-threaded mode.
* After f.use() main will be threaded and we'll have one additional thread
* with its own TLS data.
*/
ATF_TEST_CASE_WITHOUT_HEAD(cxx__thread_local_before);
ATF_TEST_CASE_BODY(cxx__thread_local_before)
{
static const char out_log[] = "Created\nCreated\nUsed\nCreated\n"
"Created\nUsed\nCreated\nDIED\nDestroyed\nDestroyed\nDestroyed\n";
ATF_REQUIRE((output = fopen("test_before.txt", "w")) != NULL);
f.use();
std::thread t([]() { f.use(); });
t.join();
fflush(output);
ATF_REQUIRE(atf::utils::compare_file("test_before.txt", out_log));
}
/*
* In this test, f.use() will test __cxa_thread_atexit()
* in threaded mode (but still in main-threaed).
*/
ATF_TEST_CASE_WITHOUT_HEAD(cxx__thread_local_after);
ATF_TEST_CASE_BODY(cxx__thread_local_after)
{
static const char out_log[] = "Created\nCreated\nUsed\nCreated\n"
"DIED\nDestroyed\nDestroyed\nDestroyed\nCreated\nCreated\nUsed\n";
ATF_REQUIRE((output = fopen("test_after.txt", "w")) != NULL);
std::thread t([]() { g.use(); });
t.join();
sleep(1);
g.use();
fflush(output);
ATF_REQUIRE(atf::utils::compare_file("test_after.txt", out_log));
}
/*
* In this test, we register a new dtor while dtors are being run
* in __cxa_thread_atexit().
*/
ATF_TEST_CASE_WITHOUT_HEAD(cxx__thread_local_add_while_calling_dtors);
ATF_TEST_CASE_BODY(cxx__thread_local_add_while_calling_dtors)
{
static const char out_log[] = "Created\nCreated\nCreated\nDIED\n"
"Destroyed\nDestroyed\nDestroyed\n";
ATF_REQUIRE((output = fopen("test_add_meanwhile.txt", "w")) != NULL);
std::thread t([]() { h.use(); });
t.join();
sleep(1);
fflush(output);
ATF_REQUIRE(atf::utils::compare_file("test_add_meanwhile.txt", out_log));
}
ATF_TEST_CASE_WITHOUT_HEAD(cxx__thread_inf_dtors);
ATF_TEST_CASE_BODY(cxx__thread_inf_dtors)
{
/*
* Only added to make isolated run of this test not
* coredumping. Construction of Foo objects require filled
* output.
*/
output = stderr;
std::thread t([]() { e.use(); });
t.join();
}
ATF_INIT_TEST_CASES(tcs)
{
ATF_ADD_TEST_CASE(tcs, cxx__thr);
ATF_ADD_TEST_CASE(tcs, cxx__thread_local_before);
ATF_ADD_TEST_CASE(tcs, cxx__thread_local_after);
ATF_ADD_TEST_CASE(tcs, cxx__thread_local_add_while_calling_dtors);
ATF_ADD_TEST_CASE(tcs, cxx__thread_inf_dtors);
}

View File

@ -153,8 +153,12 @@ thread_unwind_stop(int version, _Unwind_Action actions,
__pthread_cleanup_pop_imp(1);
}
if (done)
if (done) {
/* Tell libc that it should call non-trivial TLS dtors. */
__cxa_thread_call_dtors();
exit_thread(); /* Never return! */
}
return (_URC_NO_REASON);
}
@ -258,6 +262,8 @@ _pthread_exit_mask(void *status, sigset_t *mask)
while (curthread->cleanup != NULL) {
__pthread_cleanup_pop_imp(1);
}
__cxa_thread_call_dtors();
exit_thread();
}
@ -265,6 +271,7 @@ _pthread_exit_mask(void *status, sigset_t *mask)
while (curthread->cleanup != NULL) {
__pthread_cleanup_pop_imp(1);
}
__cxa_thread_call_dtors();
exit_thread();
#endif /* _PTHREAD_FORCED_UNWIND */